From b4c38b7edc68a3a5b324ee96518d2843e3bcec75 Mon Sep 17 00:00:00 2001 From: stevensJourney <51082125+stevensJourney@users.noreply.github.com> Date: Wed, 10 Jul 2024 18:18:08 +0200 Subject: [PATCH 01/90] [Feature] Web Navigator Locks (#54) Added Navigator locks for web mutexes --- .github/workflows/release.yml | 4 +- CHANGELOG.md | 32 +++++ README.md | 2 +- packages/drift_sqlite_async/CHANGELOG.md | 4 + packages/drift_sqlite_async/pubspec.yaml | 4 +- packages/sqlite_async/CHANGELOG.md | 4 + .../sqlite_async/lib/src/common/mutex.dart | 10 +- .../sqlite_async/lib/src/impl/stub_mutex.dart | 4 + .../lib/src/native/native_isolate_mutex.dart | 7 +- .../sqlite_async/lib/src/web/web_mutex.dart | 135 +++++++++++++++++- .../lib/src/web/web_sqlite_open_factory.dart | 2 +- packages/sqlite_async/pubspec.yaml | 3 +- packages/sqlite_async/test/mutex_test.dart | 116 ++++++++------- .../test/native/native_mutex_test.dart | 67 +++++++++ .../sqlite_async/test/web/web_mutex_test.dart | 29 ++++ 15 files changed, 355 insertions(+), 68 deletions(-) create mode 100644 CHANGELOG.md create mode 100644 packages/sqlite_async/test/native/native_mutex_test.dart create mode 100644 packages/sqlite_async/test/web/web_mutex_test.dart diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 356835d..aa26a73 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -4,8 +4,8 @@ name: Compile Assets and Create Draft Release on: push: tags: - # Trigger on tags beginning with 'v' - - 'v*' + # Trigger on sqlite_async tags + - 'sqlite_async-v*' jobs: release: diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..7ed3924 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,32 @@ +# Change Log + +All notable changes to this project will be documented in this file. +See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. + +## 2024-07-10 + +### Changes + +--- + +Packages with breaking changes: + + - There are no breaking changes in this release. + +Packages with other changes: + + - [`sqlite_async` - `v0.8.1`](#sqlite_async---v081) + - [`drift_sqlite_async` - `v0.1.0-alpha.3`](#drift_sqlite_async---v010-alpha3) + +Packages with dependency updates only: + +> Packages listed below depend on other packages in this workspace that have had changes. Their versions have been incremented to bump the minimum dependency versions of the packages they depend upon in this project. + + - `drift_sqlite_async` - `v0.1.0-alpha.3` + +--- + +#### `sqlite_async` - `v0.8.1` + + - **FEAT**: use navigator locks. + diff --git a/README.md b/README.md index dbb27aa..e3e15bf 100644 --- a/README.md +++ b/README.md @@ -13,4 +13,4 @@ This monorepo uses [melos](https://melos.invertase.dev/) to handle command and p To configure the monorepo for development run `melos prepare` after cloning. -For detailed usage, check out the inner [sqlite_async](https://github.com/powersync-ja/sqlite_async.dart/tree/main/packages/sqlite_async) and [drift_sqlite_async](https://github.com/powersync-ja/sqlite_async.dart/tree/main/packages/drift_sqlite_async) packages. +For detailed usage, check out the inner [sqlite_async](https://github.com/powersync-ja/sqlite_async.dart/tree/main/packages/sqlite_async) and [drift_sqlite_async](https://github.com/powersync-ja/sqlite_async.dart/tree/main/packages/drift_sqlite_async) packages. \ No newline at end of file diff --git a/packages/drift_sqlite_async/CHANGELOG.md b/packages/drift_sqlite_async/CHANGELOG.md index ce4d763..11d4043 100644 --- a/packages/drift_sqlite_async/CHANGELOG.md +++ b/packages/drift_sqlite_async/CHANGELOG.md @@ -1,3 +1,7 @@ +## 0.1.0-alpha.3 + + - Update a dependency to the latest release. + ## 0.1.0-alpha.2 - Update dependency `sqlite_async` to version 0.8.0. diff --git a/packages/drift_sqlite_async/pubspec.yaml b/packages/drift_sqlite_async/pubspec.yaml index 42d998c..66dc9bf 100644 --- a/packages/drift_sqlite_async/pubspec.yaml +++ b/packages/drift_sqlite_async/pubspec.yaml @@ -1,5 +1,5 @@ name: drift_sqlite_async -version: 0.1.0-alpha.2 +version: 0.1.0-alpha.3 homepage: https://github.com/powersync-ja/sqlite_async.dart repository: https://github.com/powersync-ja/sqlite_async.dart description: Use Drift with a sqlite_async database, allowing both to be used in the same application. @@ -15,7 +15,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: drift: ^2.15.0 - sqlite_async: ^0.8.0 + sqlite_async: ^0.8.1 dev_dependencies: build_runner: ^2.4.8 drift_dev: ^2.15.0 diff --git a/packages/sqlite_async/CHANGELOG.md b/packages/sqlite_async/CHANGELOG.md index e38d5cb..cd48d89 100644 --- a/packages/sqlite_async/CHANGELOG.md +++ b/packages/sqlite_async/CHANGELOG.md @@ -1,3 +1,7 @@ +## 0.8.1 + + - Added Navigator locks for web `Mutex`s. + ## 0.8.0 - Added web support (web functionality is in beta) diff --git a/packages/sqlite_async/lib/src/common/mutex.dart b/packages/sqlite_async/lib/src/common/mutex.dart index ccdcc49..edcdd49 100644 --- a/packages/sqlite_async/lib/src/common/mutex.dart +++ b/packages/sqlite_async/lib/src/common/mutex.dart @@ -1,8 +1,14 @@ import 'package:sqlite_async/src/impl/mutex_impl.dart'; abstract class Mutex { - factory Mutex() { - return MutexImpl(); + factory Mutex( + { + /// An optional identifier for this Mutex instance. + /// This could be used for platform specific logic or debugging purposes. + /// Currently this is not used on native platforms. + /// On web this will be used for the lock name if Navigator locks are available. + String? identifier}) { + return MutexImpl(identifier: identifier); } /// timeout is a timeout for acquiring the lock, not for the callback diff --git a/packages/sqlite_async/lib/src/impl/stub_mutex.dart b/packages/sqlite_async/lib/src/impl/stub_mutex.dart index 1e700fa..aefb9e6 100644 --- a/packages/sqlite_async/lib/src/impl/stub_mutex.dart +++ b/packages/sqlite_async/lib/src/impl/stub_mutex.dart @@ -1,6 +1,10 @@ import 'package:sqlite_async/src/common/mutex.dart'; class MutexImpl implements Mutex { + String? identifier; + + MutexImpl({this.identifier}); + @override Future close() { throw UnimplementedError(); diff --git a/packages/sqlite_async/lib/src/native/native_isolate_mutex.dart b/packages/sqlite_async/lib/src/native/native_isolate_mutex.dart index 23e5443..7d82f68 100644 --- a/packages/sqlite_async/lib/src/native/native_isolate_mutex.dart +++ b/packages/sqlite_async/lib/src/native/native_isolate_mutex.dart @@ -7,8 +7,8 @@ import 'package:sqlite_async/src/common/mutex.dart'; import 'package:sqlite_async/src/common/port_channel.dart'; abstract class MutexImpl implements Mutex { - factory MutexImpl() { - return SimpleMutex(); + factory MutexImpl({String? identifier}) { + return SimpleMutex(identifier: identifier); } } @@ -19,12 +19,13 @@ class SimpleMutex implements MutexImpl { // Adapted from https://github.com/tekartik/synchronized.dart/blob/master/synchronized/lib/src/basic_lock.dart Future? last; + String? identifier; // Hack to make sure the Mutex is not copied to another isolate. // ignore: unused_field final Finalizer _f = Finalizer((_) {}); - SimpleMutex(); + SimpleMutex({this.identifier}); bool get locked => last != null; diff --git a/packages/sqlite_async/lib/src/web/web_mutex.dart b/packages/sqlite_async/lib/src/web/web_mutex.dart index b5722c5..4013201 100644 --- a/packages/sqlite_async/lib/src/web/web_mutex.dart +++ b/packages/sqlite_async/lib/src/web/web_mutex.dart @@ -1,13 +1,37 @@ +import 'dart:async'; +import 'dart:math'; + +import 'package:meta/meta.dart'; import 'package:mutex/mutex.dart' as mutex; +import 'dart:js_interop'; +import 'dart:js_util' as js_util; +// This allows for checking things like hasProperty without the need for depending on the `js` package +import 'dart:js_interop_unsafe'; +import 'package:web/web.dart'; + import 'package:sqlite_async/src/common/mutex.dart'; +@JS('navigator') +external Navigator get _navigator; + /// Web implementation of [Mutex] -/// This should use `navigator.locks` in future class MutexImpl implements Mutex { - late final mutex.Mutex m; + late final mutex.Mutex fallback; + String? identifier; + final String _resolvedIdentifier; - MutexImpl() { - m = mutex.Mutex(); + MutexImpl({this.identifier}) + + /// On web a lock name is required for Navigator locks. + /// Having exclusive Mutex instances requires a somewhat unique lock name. + /// This provides a best effort unique identifier, if no identifier is provided. + /// This should be fine for most use cases: + /// - The uuid package could be added for better uniqueness if required. + /// This would add another package dependency to `sqlite_async` which is potentially unnecessary at this point. + /// An identifier should be supplied for better exclusion. + : _resolvedIdentifier = identifier ?? + "${DateTime.now().microsecondsSinceEpoch}-${Random().nextDouble()}" { + fallback = mutex.Mutex(); } @override @@ -17,8 +41,95 @@ class MutexImpl implements Mutex { @override Future lock(Future Function() callback, {Duration? timeout}) { - // Note this lock is only valid in a single web tab - return m.protect(callback); + if ((_navigator as JSObject).hasProperty('locks'.toJS).toDart) { + return _webLock(callback, timeout: timeout); + } else { + return _fallbackLock(callback, timeout: timeout); + } + } + + /// Locks the callback with a standard Mutex from the `mutex` package + Future _fallbackLock(Future Function() callback, + {Duration? timeout}) { + final completer = Completer(); + // Need to implement timeout manually for this + bool isTimedOut = false; + Timer? timer; + if (timeout != null) { + timer = Timer(timeout, () { + isTimedOut = true; + completer + .completeError(TimeoutException('Failed to acquire lock', timeout)); + }); + } + + fallback.protect(() async { + try { + if (isTimedOut) { + // Don't actually run logic + return; + } + timer?.cancel(); + final result = await callback(); + completer.complete(result); + } catch (ex) { + completer.completeError(ex); + } + }); + + return completer.future; + } + + /// Locks the callback with web Navigator locks + Future _webLock(Future Function() callback, + {Duration? timeout}) async { + final lock = await _getWebLock(timeout); + try { + final result = await callback(); + return result; + } finally { + lock.release(); + } + } + + /// Passing the Dart callback directly to the JS Navigator can cause some weird + /// context related bugs. Instead the JS lock callback will return a hold on the lock + /// which is represented as a [HeldLock]. This hold can be used when wrapping the Dart + /// callback to manage the JS lock. + /// This is inspired and adapted from https://github.com/simolus3/sqlite3.dart/blob/7bdca77afd7be7159dbef70fd1ac5aa4996211a9/sqlite3_web/lib/src/locks.dart#L6 + Future _getWebLock(Duration? timeout) { + final gotLock = Completer.sync(); + // Navigator locks can be timed out by using an AbortSignal + final controller = AbortController(); + + Timer? timer; + + if (timeout != null) { + timer = Timer(timeout, () { + gotLock + .completeError(TimeoutException('Failed to acquire lock', timeout)); + controller.abort('Timeout'.toJS); + }); + } + + // If timeout occurred before the lock is available, then this callback should not be called. + JSPromise jsCallback(JSAny lock) { + timer?.cancel(); + + // Give the Held lock something to mark this Navigator lock as completed + final jsCompleter = Completer.sync(); + gotLock.complete(HeldLock._(jsCompleter)); + return jsCompleter.future.toJS; + } + + final lockOptions = JSObject(); + lockOptions['signal'] = controller.signal; + final promise = _navigator.locks + .request(_resolvedIdentifier, lockOptions, jsCallback.toJS); + // A timeout abort will throw an exception which needs to be handled. + // There should not be any other unhandled lock errors. + js_util.promiseToFuture(promise).catchError((error) {}); + return gotLock.future; } @override @@ -26,3 +137,15 @@ class MutexImpl implements Mutex { return this; } } + +/// This represents a hold on an active Navigator lock. +/// This is created inside the Navigator lock callback function and is used to release the lock +/// from an external source. +@internal +class HeldLock { + final Completer _completer; + + HeldLock._(this._completer); + + void release() => _completer.complete(); +} diff --git a/packages/sqlite_async/lib/src/web/web_sqlite_open_factory.dart b/packages/sqlite_async/lib/src/web/web_sqlite_open_factory.dart index ce7e0c9..521320b 100644 --- a/packages/sqlite_async/lib/src/web/web_sqlite_open_factory.dart +++ b/packages/sqlite_async/lib/src/web/web_sqlite_open_factory.dart @@ -55,7 +55,7 @@ class DefaultSqliteOpenFactory // cases, we need to implement a mutex locally. final mutex = connection.access == AccessMode.throughSharedWorker ? null - : MutexImpl(); + : MutexImpl(identifier: path); // Use the DB path as a mutex identifier return WebDatabase(connection.database, options.mutex ?? mutex); } diff --git a/packages/sqlite_async/pubspec.yaml b/packages/sqlite_async/pubspec.yaml index 4523ae7..58eb5df 100644 --- a/packages/sqlite_async/pubspec.yaml +++ b/packages/sqlite_async/pubspec.yaml @@ -1,6 +1,6 @@ name: sqlite_async description: High-performance asynchronous interface for SQLite on Dart and Flutter. -version: 0.8.0 +version: 0.8.1 repository: https://github.com/powersync-ja/sqlite_async.dart environment: sdk: ">=3.4.0 <4.0.0" @@ -18,6 +18,7 @@ dependencies: collection: ^1.17.0 mutex: ^3.1.0 meta: ^1.10.0 + web: ^0.5.1 dev_dependencies: dcli: ^4.0.0 diff --git a/packages/sqlite_async/test/mutex_test.dart b/packages/sqlite_async/test/mutex_test.dart index 699a877..3f492d6 100644 --- a/packages/sqlite_async/test/mutex_test.dart +++ b/packages/sqlite_async/test/mutex_test.dart @@ -1,67 +1,83 @@ -@TestOn('!browser') -import 'dart:isolate'; +import 'dart:async'; +import 'dart:math'; -import 'package:sqlite_async/src/native/native_isolate_mutex.dart'; +import 'package:sqlite_async/sqlite_async.dart'; import 'package:test/test.dart'; +import 'utils/test_utils_impl.dart'; + +final testUtils = TestUtils(); + void main() { - group('Mutex Tests', () { - test('Closing', () async { - // Test that locks are properly released when calling SharedMutex.close() - // in in Isolate. - // A timeout in this test indicates a likely error. - for (var i = 0; i < 50; i++) { - final mutex = SimpleMutex(); - final serialized = mutex.shared; - - final result = await Isolate.run(() async { - return _lockInIsolate(serialized); + group('Shared Mutex Tests', () { + test('Queue exclusive operations', () async { + final m = Mutex(); + final collection = List.generate(10, (index) => index); + final results = []; + + final futures = collection.map((element) async { + return m.lock(() async { + // Simulate some asynchronous work + await Future.delayed(Duration(milliseconds: Random().nextInt(100))); + results.add(element); + return element; }); + }).toList(); - await mutex.lock(() async {}); + // Await all the promises + await Future.wait(futures); - expect(result, equals(5)); - } + // Check if the results are in ascending order + expect(results, equals(collection)); }); + }); - test('Re-use after closing', () async { - // Test that shared locks can be opened and closed multiple times. - final mutex = SimpleMutex(); - final serialized = mutex.shared; + test('Timeout should throw a TimeoutException', () async { + final m = Mutex(); + m.lock(() async { + await Future.delayed(Duration(milliseconds: 300)); + }); - final result = await Isolate.run(() async { - return _lockInIsolate(serialized); - }); + await expectLater( + m.lock(() async { + print('This should not get executed'); + }, timeout: Duration(milliseconds: 200)), + throwsA((e) => + e is TimeoutException && + e.message!.contains('Failed to acquire lock'))); + }); - final result2 = await Isolate.run(() async { - return _lockInIsolate(serialized); - }); + test('In-time timeout should function normally', () async { + final m = Mutex(); + final results = []; + m.lock(() async { + await Future.delayed(Duration(milliseconds: 100)); + results.add(1); + }); - await mutex.lock(() async {}); + await m.lock(() async { + results.add(2); + }, timeout: Duration(milliseconds: 200)); - expect(result, equals(5)); - expect(result2, equals(5)); - }); - }, timeout: const Timeout(Duration(milliseconds: 5000))); -} + expect(results, equals([1, 2])); + }); -Future _lockInIsolate( - SerializedMutex smutex, -) async { - final mutex = smutex.open(); - // Start a "thread" that repeatedly takes a lock - _infiniteLock(mutex).ignore(); - await Future.delayed(const Duration(milliseconds: 10)); - // Then close the mutex while the above loop is running. - await mutex.close(); - - return 5; -} + test('Different Mutex instances should cause separate locking', () async { + final m1 = Mutex(); + final m2 = Mutex(); -Future _infiniteLock(SharedMutex mutex) async { - while (true) { - await mutex.lock(() async { - await Future.delayed(const Duration(milliseconds: 1)); + final results = []; + final p1 = m1.lock(() async { + await Future.delayed(Duration(milliseconds: 300)); + results.add(1); }); - } + + final p2 = m2.lock(() async { + results.add(2); + }); + + await p1; + await p2; + expect(results, equals([2, 1])); + }); } diff --git a/packages/sqlite_async/test/native/native_mutex_test.dart b/packages/sqlite_async/test/native/native_mutex_test.dart new file mode 100644 index 0000000..699a877 --- /dev/null +++ b/packages/sqlite_async/test/native/native_mutex_test.dart @@ -0,0 +1,67 @@ +@TestOn('!browser') +import 'dart:isolate'; + +import 'package:sqlite_async/src/native/native_isolate_mutex.dart'; +import 'package:test/test.dart'; + +void main() { + group('Mutex Tests', () { + test('Closing', () async { + // Test that locks are properly released when calling SharedMutex.close() + // in in Isolate. + // A timeout in this test indicates a likely error. + for (var i = 0; i < 50; i++) { + final mutex = SimpleMutex(); + final serialized = mutex.shared; + + final result = await Isolate.run(() async { + return _lockInIsolate(serialized); + }); + + await mutex.lock(() async {}); + + expect(result, equals(5)); + } + }); + + test('Re-use after closing', () async { + // Test that shared locks can be opened and closed multiple times. + final mutex = SimpleMutex(); + final serialized = mutex.shared; + + final result = await Isolate.run(() async { + return _lockInIsolate(serialized); + }); + + final result2 = await Isolate.run(() async { + return _lockInIsolate(serialized); + }); + + await mutex.lock(() async {}); + + expect(result, equals(5)); + expect(result2, equals(5)); + }); + }, timeout: const Timeout(Duration(milliseconds: 5000))); +} + +Future _lockInIsolate( + SerializedMutex smutex, +) async { + final mutex = smutex.open(); + // Start a "thread" that repeatedly takes a lock + _infiniteLock(mutex).ignore(); + await Future.delayed(const Duration(milliseconds: 10)); + // Then close the mutex while the above loop is running. + await mutex.close(); + + return 5; +} + +Future _infiniteLock(SharedMutex mutex) async { + while (true) { + await mutex.lock(() async { + await Future.delayed(const Duration(milliseconds: 1)); + }); + } +} diff --git a/packages/sqlite_async/test/web/web_mutex_test.dart b/packages/sqlite_async/test/web/web_mutex_test.dart new file mode 100644 index 0000000..8eeefcf --- /dev/null +++ b/packages/sqlite_async/test/web/web_mutex_test.dart @@ -0,0 +1,29 @@ +import 'package:sqlite_async/sqlite_async.dart'; +import 'package:test/test.dart'; + +import '../utils/test_utils_impl.dart'; + +final testUtils = TestUtils(); + +void main() { + group('Web Mutex Tests', () { + test('Web should share locking with identical identifiers', () async { + final m1 = Mutex(identifier: 'sync'); + final m2 = Mutex(identifier: 'sync'); + + final results = []; + final p1 = m1.lock(() async { + results.add(1); + }); + + final p2 = m2.lock(() async { + results.add(2); + }); + + await p1; + await p2; + // It should be correctly ordered as if it was the same mutex + expect(results, equals([1, 2])); + }); + }); +} From 9428ad9ec49f0a12a5c0bf351b6bc56b359f99bb Mon Sep 17 00:00:00 2001 From: Mughees Khan Date: Fri, 12 Jul 2024 10:18:43 +0200 Subject: [PATCH 02/90] Drift web support (#55) * Import sqlite3_common for web * Bump version * Add web instructions for drift * Bump version * Pin drift version * Lower drift version * Drift version bounds --- CHANGELOG.md | 13 +++++----- packages/drift_sqlite_async/CHANGELOG.md | 6 ++++- packages/drift_sqlite_async/README.md | 24 +++++++++++++++++-- .../drift_sqlite_async/lib/src/executor.dart | 2 +- packages/drift_sqlite_async/pubspec.yaml | 14 ++++++++--- 5 files changed, 46 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ed3924..024eee3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,22 +11,23 @@ See [Conventional Commits](https://conventionalcommits.org) for commit guideline Packages with breaking changes: - - There are no breaking changes in this release. +- There are no breaking changes in this release. Packages with other changes: - - [`sqlite_async` - `v0.8.1`](#sqlite_async---v081) - - [`drift_sqlite_async` - `v0.1.0-alpha.3`](#drift_sqlite_async---v010-alpha3) +- [`sqlite_async` - `v0.8.1`](#sqlite_async---v081) +- [`drift_sqlite_async` - `v0.1.0-alpha.4`](#drift_sqlite_async---v010-alpha4) Packages with dependency updates only: > Packages listed below depend on other packages in this workspace that have had changes. Their versions have been incremented to bump the minimum dependency versions of the packages they depend upon in this project. - - `drift_sqlite_async` - `v0.1.0-alpha.3` +#### `drift_sqlite_async` - `v0.1.0-alpha.4` + +- **FEAT**: web support. --- #### `sqlite_async` - `v0.8.1` - - **FEAT**: use navigator locks. - +- **FEAT**: use navigator locks. diff --git a/packages/drift_sqlite_async/CHANGELOG.md b/packages/drift_sqlite_async/CHANGELOG.md index 11d4043..cb42533 100644 --- a/packages/drift_sqlite_async/CHANGELOG.md +++ b/packages/drift_sqlite_async/CHANGELOG.md @@ -1,6 +1,10 @@ +## 0.1.0-alpha.4 + +- Import `sqlite3_common` instead of `sqlite3` for web support. + ## 0.1.0-alpha.3 - - Update a dependency to the latest release. +- Update a dependency to the latest release. ## 0.1.0-alpha.2 diff --git a/packages/drift_sqlite_async/README.md b/packages/drift_sqlite_async/README.md index c732652..977dbe1 100644 --- a/packages/drift_sqlite_async/README.md +++ b/packages/drift_sqlite_async/README.md @@ -3,12 +3,12 @@ `drift_sqlite_async` allows using drift on an sqlite_async database - the APIs from both can be seamlessly used together in the same application. Supported functionality: + 1. All queries including select, insert, update, delete. 2. Transactions and nested transactions. 3. Table updates are propagated between sqlite_async and Drift - watching queries works using either API. 4. Select queries can run concurrently with writes and other select statements. - ## Usage Use `SqliteAsyncDriftConnection` to create a DatabaseConnection / QueryExecutor for Drift from the sqlite_async SqliteDatabase: @@ -59,4 +59,24 @@ These events are only sent while no write transaction is active. Within Drift's transactions, Drift's own update notifications will still apply for watching queries within that transaction. -Note: There is a possibility of events being duplicated. This should not have a significant impact on most applications. \ No newline at end of file +Note: There is a possibility of events being duplicated. This should not have a significant impact on most applications. + +## Web + +Note: Web support is currently in Beta. + +Web support requires Sqlite3 WASM and web worker Javascript files to be accessible. These file need to be put into the `web/` directory of your app. + +The compiled web worker files can be found in our Github [releases](https://github.com/powersync-ja/sqlite_async.dart/releases) +The `sqlite3.wasm` asset can be found [here](https://github.com/simolus3/sqlite3.dart/releases) + +In the end your `web/` directory will look like the following + +``` +web/ +├── favicon.png +├── index.html +├── manifest.json +├── db_worker.js +└── sqlite3.wasm +``` diff --git a/packages/drift_sqlite_async/lib/src/executor.dart b/packages/drift_sqlite_async/lib/src/executor.dart index a106b91..91c6f7f 100644 --- a/packages/drift_sqlite_async/lib/src/executor.dart +++ b/packages/drift_sqlite_async/lib/src/executor.dart @@ -2,7 +2,7 @@ import 'dart:async'; import 'package:drift/backends.dart'; import 'package:drift_sqlite_async/src/transaction_executor.dart'; -import 'package:sqlite_async/sqlite3.dart'; +import 'package:sqlite_async/sqlite3_common.dart'; import 'package:sqlite_async/sqlite_async.dart'; class _SqliteAsyncDelegate extends DatabaseDelegate { diff --git a/packages/drift_sqlite_async/pubspec.yaml b/packages/drift_sqlite_async/pubspec.yaml index 66dc9bf..842af6e 100644 --- a/packages/drift_sqlite_async/pubspec.yaml +++ b/packages/drift_sqlite_async/pubspec.yaml @@ -1,5 +1,5 @@ name: drift_sqlite_async -version: 0.1.0-alpha.3 +version: 0.1.0-alpha.4 homepage: https://github.com/powersync-ja/sqlite_async.dart repository: https://github.com/powersync-ja/sqlite_async.dart description: Use Drift with a sqlite_async database, allowing both to be used in the same application. @@ -14,12 +14,20 @@ topics: environment: sdk: ">=3.0.0 <4.0.0" dependencies: - drift: ^2.15.0 + drift: ">=2.15.0 <2.19.0" sqlite_async: ^0.8.1 dev_dependencies: build_runner: ^2.4.8 - drift_dev: ^2.15.0 + drift_dev: ">=2.15.0 <2.19.0" glob: ^2.1.2 sqlite3: ^2.4.0 test: ^1.25.2 test_api: ^0.7.0 + +platforms: + android: + ios: + linux: + macos: + windows: + web: From 0623505777a3f096bbe7db80fad9913d30526516 Mon Sep 17 00:00:00 2001 From: Christiaan Landman Date: Tue, 20 Aug 2024 11:05:23 +0200 Subject: [PATCH 03/90] Added `refreshSchema()` (#57) * Added refreshSchema() to SqliteConnection interface. * Simplifying refreshSchema implementation by introducing exlusiveLock and refreshSchema in `sqlite_database`. * Cleanup of function comment. * Updated changelog. * Removed exclusiveLock, simplified implementation of refreshSchema. * squashed * chore(release): publish packages - sqlite_async@0.8.2 - drift_sqlite_async@0.1.0-alpha.5 * Increased pana score threshold temporarily. * Added comment to pana threshold alteration. --- CHANGELOG.md | 28 ++++++ melos.yaml | 9 +- packages/drift_sqlite_async/CHANGELOG.md | 4 + packages/drift_sqlite_async/pubspec.yaml | 4 +- packages/sqlite_async/CHANGELOG.md | 6 +- .../src/native/database/connection_pool.dart | 11 +++ .../database/native_sqlite_database.dart | 5 + .../lib/src/sqlite_connection.dart | 4 + .../sqlite_async/lib/src/sqlite_queries.dart | 5 + packages/sqlite_async/pubspec.yaml | 2 +- .../sqlite_async/test/native/schema_test.dart | 98 +++++++++++++++++++ 11 files changed, 171 insertions(+), 5 deletions(-) create mode 100644 packages/sqlite_async/test/native/schema_test.dart diff --git a/CHANGELOG.md b/CHANGELOG.md index 024eee3..3b0e6a1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,34 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## 2024-08-20 + +### Changes + +--- + +Packages with breaking changes: + + - There are no breaking changes in this release. + +Packages with other changes: + + - [`sqlite_async` - `v0.8.2`](#sqlite_async---v082) + - [`drift_sqlite_async` - `v0.1.0-alpha.5`](#drift_sqlite_async---v010-alpha5) + +Packages with dependency updates only: + +> Packages listed below depend on other packages in this workspace that have had changes. Their versions have been incremented to bump the minimum dependency versions of the packages they depend upon in this project. + + - `drift_sqlite_async` - `v0.1.0-alpha.5` + +--- + +#### `sqlite_async` - `v0.8.2` + + - **FEAT**: Added `refreshSchema()`, allowing queries and watch calls to work against updated schemas. + + ## 2024-07-10 ### Changes diff --git a/melos.yaml b/melos.yaml index 19d45ad..627a740 100644 --- a/melos.yaml +++ b/melos.yaml @@ -3,6 +3,12 @@ name: sqlite_async_monorepo packages: - packages/** +command: + version: + changelog: false + packageFilters: + noPrivate: true + scripts: prepare: melos bootstrap && melos prepare:compile:webworker && melos prepare:sqlite:wasm @@ -26,9 +32,10 @@ scripts: description: Analyze Dart code in packages. run: dart analyze packages --fatal-infos + # TODO: Temporarily setting the exit-code-threshold to 10 until drift_sqlite_async dependencies are updated. analyze:packages:pana: description: Analyze Dart packages with Pana - exec: dart pub global run pana --no-warning --exit-code-threshold 0 + exec: dart pub global run pana --no-warning --exit-code-threshold 10 packageFilters: noPrivate: true diff --git a/packages/drift_sqlite_async/CHANGELOG.md b/packages/drift_sqlite_async/CHANGELOG.md index cb42533..c2b7079 100644 --- a/packages/drift_sqlite_async/CHANGELOG.md +++ b/packages/drift_sqlite_async/CHANGELOG.md @@ -1,3 +1,7 @@ +## 0.1.0-alpha.5 + + - Update a dependency to the latest release. + ## 0.1.0-alpha.4 - Import `sqlite3_common` instead of `sqlite3` for web support. diff --git a/packages/drift_sqlite_async/pubspec.yaml b/packages/drift_sqlite_async/pubspec.yaml index 842af6e..ccef204 100644 --- a/packages/drift_sqlite_async/pubspec.yaml +++ b/packages/drift_sqlite_async/pubspec.yaml @@ -1,5 +1,5 @@ name: drift_sqlite_async -version: 0.1.0-alpha.4 +version: 0.1.0-alpha.5 homepage: https://github.com/powersync-ja/sqlite_async.dart repository: https://github.com/powersync-ja/sqlite_async.dart description: Use Drift with a sqlite_async database, allowing both to be used in the same application. @@ -15,7 +15,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: drift: ">=2.15.0 <2.19.0" - sqlite_async: ^0.8.1 + sqlite_async: ^0.8.2 dev_dependencies: build_runner: ^2.4.8 drift_dev: ">=2.15.0 <2.19.0" diff --git a/packages/sqlite_async/CHANGELOG.md b/packages/sqlite_async/CHANGELOG.md index cd48d89..610273b 100644 --- a/packages/sqlite_async/CHANGELOG.md +++ b/packages/sqlite_async/CHANGELOG.md @@ -1,6 +1,10 @@ +## 0.8.2 + +- **FEAT**: Added `refreshSchema()`, allowing queries and watch calls to work against updated schemas. + ## 0.8.1 - - Added Navigator locks for web `Mutex`s. +- Added Navigator locks for web `Mutex`s. ## 0.8.0 diff --git a/packages/sqlite_async/lib/src/native/database/connection_pool.dart b/packages/sqlite_async/lib/src/native/database/connection_pool.dart index 56d9c12..9521b34 100644 --- a/packages/sqlite_async/lib/src/native/database/connection_pool.dart +++ b/packages/sqlite_async/lib/src/native/database/connection_pool.dart @@ -221,6 +221,17 @@ class SqliteConnectionPool with SqliteQueries implements SqliteConnection { // read-only connections first. await _writeConnection?.close(); } + + @override + Future refreshSchema() async { + final toRefresh = _allReadConnections.toList(); + + await _writeConnection?.refreshSchema(); + + for (var connection in toRefresh) { + await connection.refreshSchema(); + } + } } typedef ReadCallback = Future Function(SqliteReadContext tx); diff --git a/packages/sqlite_async/lib/src/native/database/native_sqlite_database.dart b/packages/sqlite_async/lib/src/native/database/native_sqlite_database.dart index 998fb79..5cb60f3 100644 --- a/packages/sqlite_async/lib/src/native/database/native_sqlite_database.dart +++ b/packages/sqlite_async/lib/src/native/database/native_sqlite_database.dart @@ -166,4 +166,9 @@ class SqliteDatabaseImpl readOnly: false, openFactory: openFactory); } + + @override + Future refreshSchema() { + return _pool.refreshSchema(); + } } diff --git a/packages/sqlite_async/lib/src/sqlite_connection.dart b/packages/sqlite_async/lib/src/sqlite_connection.dart index f92d318..f1b721a 100644 --- a/packages/sqlite_async/lib/src/sqlite_connection.dart +++ b/packages/sqlite_async/lib/src/sqlite_connection.dart @@ -130,6 +130,10 @@ abstract class SqliteConnection extends SqliteWriteContext { Future close(); + /// Ensures that all connections are aware of the latest schema changes applied (if any). + /// Queries and watch calls can potentially use outdated schema information after a schema update. + Future refreshSchema(); + /// Returns true if the connection is closed @override bool get closed; diff --git a/packages/sqlite_async/lib/src/sqlite_queries.dart b/packages/sqlite_async/lib/src/sqlite_queries.dart index d0eab7a..f777fc9 100644 --- a/packages/sqlite_async/lib/src/sqlite_queries.dart +++ b/packages/sqlite_async/lib/src/sqlite_queries.dart @@ -137,4 +137,9 @@ mixin SqliteQueries implements SqliteWriteContext, SqliteConnection { return tx.executeBatch(sql, parameterSets); }); } + + @override + Future refreshSchema() { + return get("PRAGMA table_info('sqlite_master')"); + } } diff --git a/packages/sqlite_async/pubspec.yaml b/packages/sqlite_async/pubspec.yaml index 58eb5df..e151aa8 100644 --- a/packages/sqlite_async/pubspec.yaml +++ b/packages/sqlite_async/pubspec.yaml @@ -1,6 +1,6 @@ name: sqlite_async description: High-performance asynchronous interface for SQLite on Dart and Flutter. -version: 0.8.1 +version: 0.8.2 repository: https://github.com/powersync-ja/sqlite_async.dart environment: sdk: ">=3.4.0 <4.0.0" diff --git a/packages/sqlite_async/test/native/schema_test.dart b/packages/sqlite_async/test/native/schema_test.dart new file mode 100644 index 0000000..c358402 --- /dev/null +++ b/packages/sqlite_async/test/native/schema_test.dart @@ -0,0 +1,98 @@ +@TestOn('!browser') +import 'dart:async'; + +import 'package:sqlite_async/sqlite_async.dart'; +import 'package:sqlite_async/src/utils/shared_utils.dart'; +import 'package:test/test.dart'; + +import '../utils/test_utils_impl.dart'; + +final testUtils = TestUtils(); + +void main() { + group('Schema Tests', () { + late String path; + + setUp(() async { + path = testUtils.dbPath(); + await testUtils.cleanDb(path: path); + }); + + tearDown(() async { + await testUtils.cleanDb(path: path); + }); + + createTables(SqliteDatabase db) async { + await db.writeTransaction((tx) async { + await tx.execute( + 'CREATE TABLE _customers(id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT)'); + await tx.execute( + 'CREATE TABLE _local_customers(id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT)'); + await tx + .execute('CREATE VIEW customers AS SELECT * FROM _local_customers'); + }); + } + + updateTables(SqliteDatabase db) async { + await db.writeTransaction((tx) async { + await tx.execute('DROP VIEW IF EXISTS customers'); + await tx.execute('CREATE VIEW customers AS SELECT * FROM _customers'); + }); + } + + test('should refresh schema views', () async { + final db = await testUtils.setupDatabase(path: path); + await createTables(db); + + final customerTables = + await getSourceTables(db, "select * from customers"); + expect(customerTables.contains('_local_customers'), true); + await updateTables(db); + + // without this, source tables are outdated + await db.refreshSchema(); + + final updatedCustomerTables = + await getSourceTables(db, "select * from customers"); + expect(updatedCustomerTables.contains('_customers'), true); + }); + + test('should complete refresh schema after transaction', () async { + var completer1 = Completer(); + var transactionCompleted = false; + + final db = await testUtils.setupDatabase(path: path); + await createTables(db); + + // Start a read transaction + db.readTransaction((tx) async { + completer1.complete(); + await tx.get('select test_sleep(2000)'); + + transactionCompleted = true; + }); + + // Wait for the transaction to start + await completer1.future; + + var refreshSchemaFuture = db.refreshSchema(); + + // Setup check that refreshSchema completes after the transaction has completed + var refreshAfterTransaction = false; + refreshSchemaFuture.then((_) { + if (transactionCompleted) { + refreshAfterTransaction = true; + } + }); + + await refreshSchemaFuture; + + expect(refreshAfterTransaction, isTrue, + reason: 'refreshSchema completed before transaction finished'); + + // Sanity check + expect(transactionCompleted, isTrue, + reason: 'Transaction did not complete as expected'); + }); + }); +} From 3c2b4d26038106cd0fc6bbb43480a1e18ffef49e Mon Sep 17 00:00:00 2001 From: Christiaan Landman Date: Wed, 21 Aug 2024 12:53:17 +0200 Subject: [PATCH 04/90] [Fix] `refreshSchema` not working correctly in web (#58) * Updated web database implementation to match native implementation for get/getOptional. Using getAll for refreshSchema * chore(release): publish packages - sqlite_async@0.8.3 - drift_sqlite_async@0.1.0-alpha.6 --- CHANGELOG.md | 28 +++++++++++++++++++ packages/drift_sqlite_async/CHANGELOG.md | 4 +++ packages/drift_sqlite_async/pubspec.yaml | 4 +-- packages/sqlite_async/CHANGELOG.md | 4 +++ .../sqlite_async/lib/src/sqlite_queries.dart | 2 +- .../sqlite_async/lib/src/web/database.dart | 4 +-- packages/sqlite_async/pubspec.yaml | 2 +- 7 files changed, 42 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b0e6a1..f11e596 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,34 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## 2024-08-21 + +### Changes + +--- + +Packages with breaking changes: + + - There are no breaking changes in this release. + +Packages with other changes: + + - [`sqlite_async` - `v0.8.3`](#sqlite_async---v083) + - [`drift_sqlite_async` - `v0.1.0-alpha.6`](#drift_sqlite_async---v010-alpha6) + +Packages with dependency updates only: + +> Packages listed below depend on other packages in this workspace that have had changes. Their versions have been incremented to bump the minimum dependency versions of the packages they depend upon in this project. + + - `drift_sqlite_async` - `v0.1.0-alpha.6` + +--- + +#### `sqlite_async` - `v0.8.3` + + - Updated web database implementation for get and getOptional. Fixed refreshSchema not working in web. + + ## 2024-08-20 ### Changes diff --git a/packages/drift_sqlite_async/CHANGELOG.md b/packages/drift_sqlite_async/CHANGELOG.md index c2b7079..6f377c2 100644 --- a/packages/drift_sqlite_async/CHANGELOG.md +++ b/packages/drift_sqlite_async/CHANGELOG.md @@ -1,3 +1,7 @@ +## 0.1.0-alpha.6 + + - Update a dependency to the latest release. + ## 0.1.0-alpha.5 - Update a dependency to the latest release. diff --git a/packages/drift_sqlite_async/pubspec.yaml b/packages/drift_sqlite_async/pubspec.yaml index ccef204..ca5550d 100644 --- a/packages/drift_sqlite_async/pubspec.yaml +++ b/packages/drift_sqlite_async/pubspec.yaml @@ -1,5 +1,5 @@ name: drift_sqlite_async -version: 0.1.0-alpha.5 +version: 0.1.0-alpha.6 homepage: https://github.com/powersync-ja/sqlite_async.dart repository: https://github.com/powersync-ja/sqlite_async.dart description: Use Drift with a sqlite_async database, allowing both to be used in the same application. @@ -15,7 +15,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: drift: ">=2.15.0 <2.19.0" - sqlite_async: ^0.8.2 + sqlite_async: ^0.8.3 dev_dependencies: build_runner: ^2.4.8 drift_dev: ">=2.15.0 <2.19.0" diff --git a/packages/sqlite_async/CHANGELOG.md b/packages/sqlite_async/CHANGELOG.md index 610273b..3934d0d 100644 --- a/packages/sqlite_async/CHANGELOG.md +++ b/packages/sqlite_async/CHANGELOG.md @@ -1,3 +1,7 @@ +## 0.8.3 + + - Updated web database implementation for get and getOptional. Fixed refreshSchema not working in web. + ## 0.8.2 - **FEAT**: Added `refreshSchema()`, allowing queries and watch calls to work against updated schemas. diff --git a/packages/sqlite_async/lib/src/sqlite_queries.dart b/packages/sqlite_async/lib/src/sqlite_queries.dart index f777fc9..dc91dd1 100644 --- a/packages/sqlite_async/lib/src/sqlite_queries.dart +++ b/packages/sqlite_async/lib/src/sqlite_queries.dart @@ -140,6 +140,6 @@ mixin SqliteQueries implements SqliteWriteContext, SqliteConnection { @override Future refreshSchema() { - return get("PRAGMA table_info('sqlite_master')"); + return getAll("PRAGMA table_info('sqlite_master')"); } } diff --git a/packages/sqlite_async/lib/src/web/database.dart b/packages/sqlite_async/lib/src/web/database.dart index d7b78bf..b632aa7 100644 --- a/packages/sqlite_async/lib/src/web/database.dart +++ b/packages/sqlite_async/lib/src/web/database.dart @@ -150,7 +150,7 @@ class _SharedContext implements SqliteReadContext { @override Future get(String sql, [List parameters = const []]) async { final results = await getAll(sql, parameters); - return results.single; + return results.first; } @override @@ -169,7 +169,7 @@ class _SharedContext implements SqliteReadContext { Future getOptional(String sql, [List parameters = const []]) async { final results = await getAll(sql, parameters); - return results.singleOrNull; + return results.firstOrNull; } void markClosed() { diff --git a/packages/sqlite_async/pubspec.yaml b/packages/sqlite_async/pubspec.yaml index e151aa8..088494e 100644 --- a/packages/sqlite_async/pubspec.yaml +++ b/packages/sqlite_async/pubspec.yaml @@ -1,6 +1,6 @@ name: sqlite_async description: High-performance asynchronous interface for SQLite on Dart and Flutter. -version: 0.8.2 +version: 0.8.3 repository: https://github.com/powersync-ja/sqlite_async.dart environment: sdk: ">=3.4.0 <4.0.0" From 9e4feb8cca7e2e09283ea4291352d9a1e00e5590 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Tue, 3 Sep 2024 10:50:49 +0200 Subject: [PATCH 05/90] Support latest version of package:web (#59) --- packages/sqlite_async/pubspec.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/sqlite_async/pubspec.yaml b/packages/sqlite_async/pubspec.yaml index 088494e..e87eab7 100644 --- a/packages/sqlite_async/pubspec.yaml +++ b/packages/sqlite_async/pubspec.yaml @@ -13,12 +13,12 @@ topics: dependencies: sqlite3: "^2.4.4" - sqlite3_web: ^0.1.2-wip + sqlite3_web: ^0.1.3 async: ^2.10.0 collection: ^1.17.0 mutex: ^3.1.0 meta: ^1.10.0 - web: ^0.5.1 + web: ^1.0.0 dev_dependencies: dcli: ^4.0.0 From eb144be3c6bc60fa0cbf04568dafa3a4b8231a32 Mon Sep 17 00:00:00 2001 From: Mughees Khan Date: Tue, 3 Sep 2024 15:51:00 +0200 Subject: [PATCH 06/90] Export sqlite3 open (#60) * chore: Export sqlite3_open * Update pana threshold * chore(release): publish packages - sqlite_async@0.9.0 - drift_sqlite_async@0.1.0-alpha.7 --- CHANGELOG.md | 30 +++++++++++++++++++++ melos.yaml | 4 +-- packages/drift_sqlite_async/CHANGELOG.md | 4 +++ packages/drift_sqlite_async/pubspec.yaml | 4 +-- packages/sqlite_async/CHANGELOG.md | 6 +++++ packages/sqlite_async/lib/sqlite3_open.dart | 3 +++ packages/sqlite_async/pubspec.yaml | 2 +- 7 files changed, 48 insertions(+), 5 deletions(-) create mode 100644 packages/sqlite_async/lib/sqlite3_open.dart diff --git a/CHANGELOG.md b/CHANGELOG.md index f11e596..19e76bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,36 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## 2024-09-03 + +### Changes + +--- + +Packages with breaking changes: + + - There are no breaking changes in this release. + +Packages with other changes: + + - [`sqlite_async` - `v0.9.0`](#sqlite_async---v090) + - [`drift_sqlite_async` - `v0.1.0-alpha.7`](#drift_sqlite_async---v010-alpha7) + +Packages with dependency updates only: + +> Packages listed below depend on other packages in this workspace that have had changes. Their versions have been incremented to bump the minimum dependency versions of the packages they depend upon in this project. + + - `drift_sqlite_async` - `v0.1.0-alpha.7` + +--- + +#### `sqlite_async` - `v0.9.0` + + - Support the latest version of package:web and package:sqlite3_web + + - Export sqlite3 `open` for packages that depend on `sqlite_async` + + ## 2024-08-21 ### Changes diff --git a/melos.yaml b/melos.yaml index 627a740..8de8bfe 100644 --- a/melos.yaml +++ b/melos.yaml @@ -32,10 +32,10 @@ scripts: description: Analyze Dart code in packages. run: dart analyze packages --fatal-infos - # TODO: Temporarily setting the exit-code-threshold to 10 until drift_sqlite_async dependencies are updated. + # TODO: Temporarily setting the exit-code-threshold to 20 until drift_sqlite_async dependencies are updated. analyze:packages:pana: description: Analyze Dart packages with Pana - exec: dart pub global run pana --no-warning --exit-code-threshold 10 + exec: dart pub global run pana --no-warning --exit-code-threshold 20 packageFilters: noPrivate: true diff --git a/packages/drift_sqlite_async/CHANGELOG.md b/packages/drift_sqlite_async/CHANGELOG.md index 6f377c2..455f9bd 100644 --- a/packages/drift_sqlite_async/CHANGELOG.md +++ b/packages/drift_sqlite_async/CHANGELOG.md @@ -1,3 +1,7 @@ +## 0.1.0-alpha.7 + + - Update a dependency to the latest release. + ## 0.1.0-alpha.6 - Update a dependency to the latest release. diff --git a/packages/drift_sqlite_async/pubspec.yaml b/packages/drift_sqlite_async/pubspec.yaml index ca5550d..1f38930 100644 --- a/packages/drift_sqlite_async/pubspec.yaml +++ b/packages/drift_sqlite_async/pubspec.yaml @@ -1,5 +1,5 @@ name: drift_sqlite_async -version: 0.1.0-alpha.6 +version: 0.1.0-alpha.7 homepage: https://github.com/powersync-ja/sqlite_async.dart repository: https://github.com/powersync-ja/sqlite_async.dart description: Use Drift with a sqlite_async database, allowing both to be used in the same application. @@ -15,7 +15,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: drift: ">=2.15.0 <2.19.0" - sqlite_async: ^0.8.3 + sqlite_async: ^0.9.0 dev_dependencies: build_runner: ^2.4.8 drift_dev: ">=2.15.0 <2.19.0" diff --git a/packages/sqlite_async/CHANGELOG.md b/packages/sqlite_async/CHANGELOG.md index 3934d0d..a91538e 100644 --- a/packages/sqlite_async/CHANGELOG.md +++ b/packages/sqlite_async/CHANGELOG.md @@ -1,3 +1,9 @@ +## 0.9.0 + + - Support the latest version of package:web and package:sqlite3_web + + - Export sqlite3 `open` for packages that depend on `sqlite_async` + ## 0.8.3 - Updated web database implementation for get and getOptional. Fixed refreshSchema not working in web. diff --git a/packages/sqlite_async/lib/sqlite3_open.dart b/packages/sqlite_async/lib/sqlite3_open.dart new file mode 100644 index 0000000..9d9b0a2 --- /dev/null +++ b/packages/sqlite_async/lib/sqlite3_open.dart @@ -0,0 +1,3 @@ +// Re-exports [sqlite3 open](https://pub.dev/packages/sqlite3) to expose sqlite3 open without +// adding it as a direct dependency. +export 'package:sqlite3/open.dart'; diff --git a/packages/sqlite_async/pubspec.yaml b/packages/sqlite_async/pubspec.yaml index e87eab7..438acbb 100644 --- a/packages/sqlite_async/pubspec.yaml +++ b/packages/sqlite_async/pubspec.yaml @@ -1,6 +1,6 @@ name: sqlite_async description: High-performance asynchronous interface for SQLite on Dart and Flutter. -version: 0.8.3 +version: 0.9.0 repository: https://github.com/powersync-ja/sqlite_async.dart environment: sdk: ">=3.4.0 <4.0.0" From a243899ea6bc7bba7a4a6870ce19f6367f8fb07e Mon Sep 17 00:00:00 2001 From: David Martos Date: Mon, 23 Sep 2024 20:21:21 +0200 Subject: [PATCH 07/90] Support passing `logStatements` to drift --- packages/drift_sqlite_async/lib/src/connection.dart | 4 ++-- packages/drift_sqlite_async/lib/src/executor.dart | 6 ++---- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/packages/drift_sqlite_async/lib/src/connection.dart b/packages/drift_sqlite_async/lib/src/connection.dart index e375795..a1af55a 100644 --- a/packages/drift_sqlite_async/lib/src/connection.dart +++ b/packages/drift_sqlite_async/lib/src/connection.dart @@ -15,8 +15,8 @@ import 'package:sqlite_async/sqlite_async.dart'; class SqliteAsyncDriftConnection extends DatabaseConnection { late StreamSubscription _updateSubscription; - SqliteAsyncDriftConnection(SqliteConnection db) - : super(SqliteAsyncQueryExecutor(db)) { + SqliteAsyncDriftConnection(SqliteConnection db, {bool logStatements = false}) + : super(SqliteAsyncQueryExecutor(db, logStatements: logStatements)) { _updateSubscription = (db as SqliteQueries).updates!.listen((event) { var setUpdates = {}; for (var tableName in event.tables) { diff --git a/packages/drift_sqlite_async/lib/src/executor.dart b/packages/drift_sqlite_async/lib/src/executor.dart index 91c6f7f..b68e2e6 100644 --- a/packages/drift_sqlite_async/lib/src/executor.dart +++ b/packages/drift_sqlite_async/lib/src/executor.dart @@ -127,10 +127,8 @@ class _SqliteAsyncVersionDelegate extends DynamicVersionDelegate { /// Extnral update notifications from the [SqliteConnection] are _not_ forwarded /// automatically - use [SqliteAsyncDriftConnection] for that. class SqliteAsyncQueryExecutor extends DelegatedDatabase { - SqliteAsyncQueryExecutor(SqliteConnection db) - : super( - _SqliteAsyncDelegate(db), - ); + SqliteAsyncQueryExecutor(SqliteConnection db, {bool logStatements = false}) + : super(_SqliteAsyncDelegate(db), logStatements: logStatements); /// The underlying SqliteConnection used by drift to send queries. SqliteConnection get db { From bffb3500bc34f8b52da5f71a12845de1e85cddf3 Mon Sep 17 00:00:00 2001 From: David Martos Date: Thu, 10 Oct 2024 13:38:19 +0200 Subject: [PATCH 08/90] Support latest version of package:web in drift adapter (#65) * Support latest version of package:web in drift adapter * Avoid using internal drift APIs * complement test * changelog --------- Co-authored-by: Simon Binder --- packages/drift_sqlite_async/CHANGELOG.md | 5 + .../drift_sqlite_async/lib/src/executor.dart | 76 +++++-- .../lib/src/transaction_executor.dart | 193 ------------------ packages/drift_sqlite_async/pubspec.yaml | 6 +- .../drift_sqlite_async/test/basic_test.dart | 9 +- 5 files changed, 70 insertions(+), 219 deletions(-) delete mode 100644 packages/drift_sqlite_async/lib/src/transaction_executor.dart diff --git a/packages/drift_sqlite_async/CHANGELOG.md b/packages/drift_sqlite_async/CHANGELOG.md index 455f9bd..168af4e 100644 --- a/packages/drift_sqlite_async/CHANGELOG.md +++ b/packages/drift_sqlite_async/CHANGELOG.md @@ -1,3 +1,8 @@ +## 0.2.0-alpha.1 + + - Support `drift` version >=2.19 and `web` 1.0.0 + - BREAKING CHANGE: Nested transactions through drift no longer create SAVEPOINTs. When nesting a drift `transaction`, the transaction is reused. See https://github.com/powersync-ja/sqlite_async.dart/pull/65 + ## 0.1.0-alpha.7 - Update a dependency to the latest release. diff --git a/packages/drift_sqlite_async/lib/src/executor.dart b/packages/drift_sqlite_async/lib/src/executor.dart index b68e2e6..5d670c1 100644 --- a/packages/drift_sqlite_async/lib/src/executor.dart +++ b/packages/drift_sqlite_async/lib/src/executor.dart @@ -1,15 +1,24 @@ import 'dart:async'; import 'package:drift/backends.dart'; -import 'package:drift_sqlite_async/src/transaction_executor.dart'; +import 'package:drift/src/runtime/query_builder/query_builder.dart'; import 'package:sqlite_async/sqlite3_common.dart'; import 'package:sqlite_async/sqlite_async.dart'; -class _SqliteAsyncDelegate extends DatabaseDelegate { +// Ends with " RETURNING *", or starts with insert/update/delete. +// Drift-generated queries will always have the RETURNING *. +// The INSERT/UPDATE/DELETE check is for custom queries, and is not exhaustive. +final _returningCheck = RegExp(r'( RETURNING \*;?$)|(^(INSERT|UPDATE|DELETE))', + caseSensitive: false); + +class _SqliteAsyncDelegate extends _SqliteAsyncQueryDelegate + implements DatabaseDelegate { final SqliteConnection db; bool _closed = false; - _SqliteAsyncDelegate(this.db); + _SqliteAsyncDelegate(this.db) : super(db, db.writeLock); + + bool isInTransaction = false; // unused @override late final DbVersionDelegate versionDelegate = @@ -18,18 +27,11 @@ class _SqliteAsyncDelegate extends DatabaseDelegate { // Not used - we override beginTransaction() with SqliteAsyncTransactionExecutor for more control. @override late final TransactionDelegate transactionDelegate = - const NoTransactionDelegate(); + _SqliteAsyncTransactionDelegate(db); @override bool get isOpen => !db.closed && !_closed; - // Ends with " RETURNING *", or starts with insert/update/delete. - // Drift-generated queries will always have the RETURNING *. - // The INSERT/UPDATE/DELETE check is for custom queries, and is not exhaustive. - final _returningCheck = RegExp( - r'( RETURNING \*;?$)|(^(INSERT|UPDATE|DELETE))', - caseSensitive: false); - @override Future open(QueryExecutorUser user) async { // Workaround - this ensures the db is open @@ -42,9 +44,30 @@ class _SqliteAsyncDelegate extends DatabaseDelegate { _closed = true; } + @override + void notifyDatabaseOpened(OpeningDetails details) { + // Unused + } +} + +class _SqliteAsyncQueryDelegate extends QueryDelegate { + final SqliteWriteContext _context; + final Future Function( + Future Function(SqliteWriteContext tx) callback)? _writeLock; + + _SqliteAsyncQueryDelegate(this._context, this._writeLock); + + Future writeLock(Future Function(SqliteWriteContext tx) callback) { + if (_writeLock case var writeLock?) { + return writeLock.call(callback); + } else { + return callback(_context); + } + } + @override Future runBatched(BatchedStatements statements) async { - return db.writeLock((tx) async { + return writeLock((tx) async { // sqlite_async's batch functionality doesn't have enough flexibility to support // this with prepared statements yet. for (final arg in statements.arguments) { @@ -56,12 +79,12 @@ class _SqliteAsyncDelegate extends DatabaseDelegate { @override Future runCustom(String statement, List args) { - return db.execute(statement, args); + return _context.execute(statement, args); } @override Future runInsert(String statement, List args) async { - return db.writeLock((tx) async { + return writeLock((tx) async { await tx.execute(statement, args); final row = await tx.get('SELECT last_insert_rowid() as row_id'); return row['row_id']; @@ -77,17 +100,17 @@ class _SqliteAsyncDelegate extends DatabaseDelegate { // This takes write lock, so we want to avoid it for plain select statements. // This is not an exhaustive check, but should cover all Drift-generated queries using // `runSelect()`. - result = await db.execute(statement, args); + result = await _context.execute(statement, args); } else { // Plain SELECT statement - use getAll() to avoid using a write lock. - result = await db.getAll(statement, args); + result = await _context.getAll(statement, args); } return QueryResult(result.columnNames, result.rows); } @override Future runUpdate(String statement, List args) { - return db.writeLock((tx) async { + return writeLock((tx) async { await tx.execute(statement, args); final row = await tx.get('SELECT changes() as changes'); return row['changes']; @@ -95,6 +118,20 @@ class _SqliteAsyncDelegate extends DatabaseDelegate { } } +class _SqliteAsyncTransactionDelegate extends SupportedTransactionDelegate { + final SqliteConnection _db; + + _SqliteAsyncTransactionDelegate(this._db); + + @override + Future startTransaction(Future Function(QueryDelegate p1) run) async { + await _db.writeTransaction((context) async { + final delegate = _SqliteAsyncQueryDelegate(context, null); + return run(delegate); + }); + } +} + class _SqliteAsyncVersionDelegate extends DynamicVersionDelegate { final SqliteConnection _db; @@ -137,9 +174,4 @@ class SqliteAsyncQueryExecutor extends DelegatedDatabase { @override bool get isSequential => false; - - @override - TransactionExecutor beginTransaction() { - return SqliteAsyncTransactionExecutor(db); - } } diff --git a/packages/drift_sqlite_async/lib/src/transaction_executor.dart b/packages/drift_sqlite_async/lib/src/transaction_executor.dart deleted file mode 100644 index ee4db9c..0000000 --- a/packages/drift_sqlite_async/lib/src/transaction_executor.dart +++ /dev/null @@ -1,193 +0,0 @@ -import 'dart:async'; - -import 'package:drift/backends.dart'; -import 'package:sqlite_async/sqlite_async.dart'; - -/// Based on Drift's _WrappingTransactionExecutor, which is private. -/// Extended to support nested transactions. -/// -/// The outer SqliteAsyncTransactionExecutor uses sqlite_async's writeTransaction, which -/// does BEGIN/COMMIT/ROLLBACK. -/// -/// Nested transactions use SqliteAsyncNestedTransactionExecutor to implement SAVEPOINT / ROLLBACK. -class SqliteAsyncTransactionExecutor extends TransactionExecutor - with _TransactionQueryMixin { - final SqliteConnection _db; - static final _artificialRollback = - Exception('artificial exception to rollback the transaction'); - final Zone _createdIn = Zone.current; - final Completer _completerForCallback = Completer(); - Completer? _opened, _finished; - - /// Whether this executor has explicitly been closed. - bool _closed = false; - - @override - late SqliteWriteContext ctx; - - SqliteAsyncTransactionExecutor(this._db); - - void _checkCanOpen() { - if (_closed) { - throw StateError( - "A tranaction was used after being closed. Please check that you're " - 'awaiting all database operations inside a `transaction` block.'); - } - } - - @override - Future ensureOpen(QueryExecutorUser user) { - _checkCanOpen(); - var opened = _opened; - - if (opened == null) { - _opened = opened = Completer(); - _createdIn.run(() async { - final result = _db.writeTransaction((innerCtx) async { - opened!.complete(); - ctx = innerCtx; - await _completerForCallback.future; - }); - - _finished = Completer() - ..complete( - // ignore: void_checks - result - // Ignore the exception caused by [rollback] which may be - // rethrown by startTransaction - .onError((error, stackTrace) => null, - test: (e) => e == _artificialRollback) - // Consider this transaction closed after the call completes - // This may happen without send/rollback being called in - // case there's an exception when opening the transaction. - .whenComplete(() => _closed = true), - ); - }); - } - - // The opened completer is never completed if `startTransaction` throws - // before our callback is invoked (probably becaue `BEGIN` threw an - // exception). In that case, _finished will complete with that error though. - return Future.any([opened.future, if (_finished != null) _finished!.future]) - .then((value) => true); - } - - @override - Future send() async { - // don't do anything if the transaction completes before it was opened - if (_opened == null || _closed) return; - - _completerForCallback.complete(); - _closed = true; - await _finished?.future; - } - - @override - Future rollback() async { - // Note: This may be called after send() if send() throws (that is, the - // transaction can't be completed). But if completing fails, we assume that - // the transaction will implicitly be rolled back the underlying connection - // (it's not like we could explicitly roll it back, we only have one - // callback to implement). - if (_opened == null || _closed) return; - - _completerForCallback.completeError(_artificialRollback); - _closed = true; - await _finished?.future; - } - - @override - TransactionExecutor beginTransaction() { - return SqliteAsyncNestedTransactionExecutor(ctx, 1); - } - - @override - SqlDialect get dialect => SqlDialect.sqlite; - - @override - bool get supportsNestedTransactions => true; -} - -class SqliteAsyncNestedTransactionExecutor extends TransactionExecutor - with _TransactionQueryMixin { - @override - final SqliteWriteContext ctx; - - int depth; - - SqliteAsyncNestedTransactionExecutor(this.ctx, this.depth); - - @override - Future ensureOpen(QueryExecutorUser user) async { - await ctx.execute('SAVEPOINT tx$depth'); - return true; - } - - @override - Future send() async { - await ctx.execute('RELEASE SAVEPOINT tx$depth'); - } - - @override - Future rollback() async { - await ctx.execute('ROLLBACK TO SAVEPOINT tx$depth'); - } - - @override - TransactionExecutor beginTransaction() { - return SqliteAsyncNestedTransactionExecutor(ctx, depth + 1); - } - - @override - SqlDialect get dialect => SqlDialect.sqlite; - - @override - bool get supportsNestedTransactions => true; -} - -abstract class _QueryDelegate { - SqliteWriteContext get ctx; -} - -mixin _TransactionQueryMixin implements QueryExecutor, _QueryDelegate { - @override - Future runBatched(BatchedStatements statements) async { - // sqlite_async's batch functionality doesn't have enough flexibility to support - // this with prepared statements yet. - for (final arg in statements.arguments) { - await ctx.execute( - statements.statements[arg.statementIndex], arg.arguments); - } - } - - @override - Future runCustom(String statement, [List? args]) { - return ctx.execute(statement, args ?? const []); - } - - @override - Future runInsert(String statement, List args) async { - await ctx.execute(statement, args); - final row = await ctx.get('SELECT last_insert_rowid() as row_id'); - return row['row_id']; - } - - @override - Future>> runSelect( - String statement, List args) async { - final result = await ctx.execute(statement, args); - return QueryResult(result.columnNames, result.rows).asMap.toList(); - } - - @override - Future runUpdate(String statement, List args) async { - await ctx.execute(statement, args); - final row = await ctx.get('SELECT changes() as changes'); - return row['changes']; - } - - @override - Future runDelete(String statement, List args) { - return runUpdate(statement, args); - } -} diff --git a/packages/drift_sqlite_async/pubspec.yaml b/packages/drift_sqlite_async/pubspec.yaml index 1f38930..b168037 100644 --- a/packages/drift_sqlite_async/pubspec.yaml +++ b/packages/drift_sqlite_async/pubspec.yaml @@ -1,5 +1,5 @@ name: drift_sqlite_async -version: 0.1.0-alpha.7 +version: 0.2.0-alpha.1 homepage: https://github.com/powersync-ja/sqlite_async.dart repository: https://github.com/powersync-ja/sqlite_async.dart description: Use Drift with a sqlite_async database, allowing both to be used in the same application. @@ -14,11 +14,11 @@ topics: environment: sdk: ">=3.0.0 <4.0.0" dependencies: - drift: ">=2.15.0 <2.19.0" + drift: ">=2.19.0 <3.0.0" sqlite_async: ^0.9.0 dev_dependencies: build_runner: ^2.4.8 - drift_dev: ">=2.15.0 <2.19.0" + drift_dev: ">=2.19.0 <3.0.0" glob: ^2.1.2 sqlite3: ^2.4.0 test: ^1.25.2 diff --git a/packages/drift_sqlite_async/test/basic_test.dart b/packages/drift_sqlite_async/test/basic_test.dart index 503604f..1b33562 100644 --- a/packages/drift_sqlite_async/test/basic_test.dart +++ b/packages/drift_sqlite_async/test/basic_test.dart @@ -129,6 +129,13 @@ void main() { // This runs outside the transaction expect(await db.get('select count(*) as count from test_data'), equals({'count': 0})); + + // This runs in the transaction + final countInTransaction = (await dbu + .customSelect('select count(*) as count from test_data') + .getSingle()) + .data; + expect(countInTransaction, equals({'count': 2})); }); expect(await db.get('select count(*) as count from test_data'), @@ -172,7 +179,7 @@ void main() { {'description': 'Test 1'}, {'description': 'Test 3'} ])); - }); + }, skip: 'sqlite_async does not support nested transactions'); test('Concurrent select', () async { var completer1 = Completer(); From 3813c9be494bf4633f765a56ee87d02813218030 Mon Sep 17 00:00:00 2001 From: Mugi Khan Date: Thu, 10 Oct 2024 13:48:35 +0200 Subject: [PATCH 09/90] chore: Update gitignore and changelog formatting --- .gitignore | 2 ++ packages/drift_sqlite_async/CHANGELOG.md | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 0bb8318..e0c7669 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,5 @@ doc *.iml build + +.DS_store \ No newline at end of file diff --git a/packages/drift_sqlite_async/CHANGELOG.md b/packages/drift_sqlite_async/CHANGELOG.md index 168af4e..e90c7b9 100644 --- a/packages/drift_sqlite_async/CHANGELOG.md +++ b/packages/drift_sqlite_async/CHANGELOG.md @@ -1,7 +1,7 @@ ## 0.2.0-alpha.1 - - Support `drift` version >=2.19 and `web` 1.0.0 - - BREAKING CHANGE: Nested transactions through drift no longer create SAVEPOINTs. When nesting a drift `transaction`, the transaction is reused. See https://github.com/powersync-ja/sqlite_async.dart/pull/65 + - Support `drift` version >=2.19 and `web` v1.0.0. + - **BREAKING CHANGE**: Nested transactions through drift no longer create SAVEPOINTs. When nesting a drift `transaction`, the transaction is reused. See https://github.com/powersync-ja/sqlite_async.dart/pull/65. ## 0.1.0-alpha.7 From c08af0ad96f7e846465c395c20df04677e500181 Mon Sep 17 00:00:00 2001 From: Mugi Khan Date: Thu, 10 Oct 2024 13:52:50 +0200 Subject: [PATCH 10/90] chore: PR link format --- packages/drift_sqlite_async/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/drift_sqlite_async/CHANGELOG.md b/packages/drift_sqlite_async/CHANGELOG.md index e90c7b9..c82b6d7 100644 --- a/packages/drift_sqlite_async/CHANGELOG.md +++ b/packages/drift_sqlite_async/CHANGELOG.md @@ -1,7 +1,7 @@ ## 0.2.0-alpha.1 - Support `drift` version >=2.19 and `web` v1.0.0. - - **BREAKING CHANGE**: Nested transactions through drift no longer create SAVEPOINTs. When nesting a drift `transaction`, the transaction is reused. See https://github.com/powersync-ja/sqlite_async.dart/pull/65. + - **BREAKING CHANGE**: Nested transactions through drift no longer create SAVEPOINTs. When nesting a drift `transaction`, the transaction is reused ([#65](https://github.com/powersync-ja/sqlite_async.dart/pull/65)). ## 0.1.0-alpha.7 From 9c23ce6f28a59c367c28bf3704113220b1a1b848 Mon Sep 17 00:00:00 2001 From: Mughees Khan Date: Thu, 10 Oct 2024 14:04:37 +0200 Subject: [PATCH 11/90] Update gitignore and changelog formatting (#67) * chore: Update gitignore and changelog formatting * chore: PR link format --- .gitignore | 2 ++ packages/drift_sqlite_async/CHANGELOG.md | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 0bb8318..e0c7669 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,5 @@ doc *.iml build + +.DS_store \ No newline at end of file diff --git a/packages/drift_sqlite_async/CHANGELOG.md b/packages/drift_sqlite_async/CHANGELOG.md index 168af4e..c82b6d7 100644 --- a/packages/drift_sqlite_async/CHANGELOG.md +++ b/packages/drift_sqlite_async/CHANGELOG.md @@ -1,7 +1,7 @@ ## 0.2.0-alpha.1 - - Support `drift` version >=2.19 and `web` 1.0.0 - - BREAKING CHANGE: Nested transactions through drift no longer create SAVEPOINTs. When nesting a drift `transaction`, the transaction is reused. See https://github.com/powersync-ja/sqlite_async.dart/pull/65 + - Support `drift` version >=2.19 and `web` v1.0.0. + - **BREAKING CHANGE**: Nested transactions through drift no longer create SAVEPOINTs. When nesting a drift `transaction`, the transaction is reused ([#65](https://github.com/powersync-ja/sqlite_async.dart/pull/65)). ## 0.1.0-alpha.7 From 6971baac8e04bdaed0d40695efcd483ca1e947aa Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Thu, 17 Oct 2024 14:00:02 +0200 Subject: [PATCH 12/90] Add failing test. --- packages/sqlite_async/test/watch_test.dart | 41 ++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/packages/sqlite_async/test/watch_test.dart b/packages/sqlite_async/test/watch_test.dart index e0ef765..b179009 100644 --- a/packages/sqlite_async/test/watch_test.dart +++ b/packages/sqlite_async/test/watch_test.dart @@ -253,5 +253,46 @@ void main() { done = true; } }); + + test('watch with transaction', () async { + final db = await testUtils.setupDatabase(path: path); + await createTables(db); + + const baseTime = 20; + + const throttleDuration = Duration(milliseconds: baseTime); + // delay must be bigger than throttleDuration, and bigger + // than any internal throttles. + const delay = Duration(milliseconds: baseTime * 3); + + final stream = db.watch('SELECT count() AS count FROM assets', + throttle: throttleDuration); + + List counts = []; + + final subscription = stream.listen((e) { + counts.add(e.first['count']); + }); + await Future.delayed(delay); + + await db.writeTransaction((tx) async { + await tx.execute('INSERT INTO assets(make) VALUES (?)', ['test1']); + await Future.delayed(delay); + await tx.execute('INSERT INTO assets(make) VALUES (?)', ['test2']); + await Future.delayed(delay); + }); + await Future.delayed(delay); + + subscription.cancel(); + + expect( + counts, + equals([ + // one event when starting the subscription + 0, + // one event after the transaction + 2 + ])); + }); }); } From 64b349eef52788c57a60f7c7ab605fbb14766781 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Thu, 17 Oct 2024 14:35:31 +0200 Subject: [PATCH 13/90] Fix update notifications that should not trigger inside transactions. --- .../database/native_sqlite_connection_impl.dart | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/packages/sqlite_async/lib/src/native/database/native_sqlite_connection_impl.dart b/packages/sqlite_async/lib/src/native/database/native_sqlite_connection_impl.dart index b7ef76b..299339c 100644 --- a/packages/sqlite_async/lib/src/native/database/native_sqlite_connection_impl.dart +++ b/packages/sqlite_async/lib/src/native/database/native_sqlite_connection_impl.dart @@ -286,7 +286,17 @@ Future _sqliteConnectionIsolateInner(_SqliteConnectionParams params, Object? txError; void maybeFireUpdates() { - if (updatedTables.isNotEmpty) { + // We keep buffering the set of updated tables until we are not + // in a transaction. Firing transactions inside a transaction + // has multiple issues: + // 1. Watched queries would detect changes to the underlying tables, + // but the data would not be visible to queries yet. + // 2. It would trigger many more notifications than required. + // + // This still includes updates for transactions that are rolled back. + // We could handle those better at a later stage. + + if (updatedTables.isNotEmpty && db.autocommit) { client.fire(UpdateNotification(updatedTables)); updatedTables.clear(); updateDebouncer?.cancel(); @@ -301,7 +311,7 @@ Future _sqliteConnectionIsolateInner(_SqliteConnectionParams params, // 1. Update arrived after _SqliteIsolateClose (not sure if this could happen). // 2. Long-running _SqliteIsolateClosure that should fire updates while running. updateDebouncer ??= - Timer(const Duration(milliseconds: 10), maybeFireUpdates); + Timer(const Duration(milliseconds: 1), maybeFireUpdates); }); server.open((data) async { From 2178d69d896ced8c16bdae055a224e7bac9c7644 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Thu, 17 Oct 2024 16:07:24 +0200 Subject: [PATCH 14/90] Throttle update notifications for web implementation. --- .../native_sqlite_connection_impl.dart | 5 +- .../web/worker/throttled_common_database.dart | 197 ++++++++++++++++++ .../lib/src/web/worker/worker_utils.dart | 5 +- packages/sqlite_async/test/watch_test.dart | 9 +- 4 files changed, 212 insertions(+), 4 deletions(-) create mode 100644 packages/sqlite_async/lib/src/web/worker/throttled_common_database.dart diff --git a/packages/sqlite_async/lib/src/native/database/native_sqlite_connection_impl.dart b/packages/sqlite_async/lib/src/native/database/native_sqlite_connection_impl.dart index 299339c..7df4ac8 100644 --- a/packages/sqlite_async/lib/src/native/database/native_sqlite_connection_impl.dart +++ b/packages/sqlite_async/lib/src/native/database/native_sqlite_connection_impl.dart @@ -299,9 +299,9 @@ Future _sqliteConnectionIsolateInner(_SqliteConnectionParams params, if (updatedTables.isNotEmpty && db.autocommit) { client.fire(UpdateNotification(updatedTables)); updatedTables.clear(); - updateDebouncer?.cancel(); - updateDebouncer = null; } + updateDebouncer?.cancel(); + updateDebouncer = null; } db.updates.listen((event) { @@ -316,6 +316,7 @@ Future _sqliteConnectionIsolateInner(_SqliteConnectionParams params, server.open((data) async { if (data is _SqliteIsolateClose) { + // This is a transaction close message if (txId != null) { if (!db.autocommit) { db.execute('ROLLBACK'); diff --git a/packages/sqlite_async/lib/src/web/worker/throttled_common_database.dart b/packages/sqlite_async/lib/src/web/worker/throttled_common_database.dart new file mode 100644 index 0000000..da69d12 --- /dev/null +++ b/packages/sqlite_async/lib/src/web/worker/throttled_common_database.dart @@ -0,0 +1,197 @@ +import 'dart:async'; + +import 'package:sqlite_async/sqlite3_common.dart'; + +/// Wrap a CommonDatabase to throttle its updates stream. +/// This is so that we can throttle the updates _within_ +/// the worker process, avoiding mass notifications over +/// the MessagePort. +class ThrottledCommonDatabase extends CommonDatabase { + final CommonDatabase _db; + final StreamController _transactionController = + StreamController.broadcast(); + + ThrottledCommonDatabase(this._db); + + @override + int get userVersion => _db.userVersion; + + @override + set userVersion(int userVersion) { + _db.userVersion = userVersion; + } + + @override + bool get autocommit => _db.autocommit; + + @override + DatabaseConfig get config => _db.config; + + @override + void createAggregateFunction( + {required String functionName, + required AggregateFunction function, + AllowedArgumentCount argumentCount = const AllowedArgumentCount.any(), + bool deterministic = false, + bool directOnly = true}) { + _db.createAggregateFunction(functionName: functionName, function: function); + } + + @override + void createCollation( + {required String name, required CollatingFunction function}) { + _db.createCollation(name: name, function: function); + } + + @override + void createFunction( + {required String functionName, + required ScalarFunction function, + AllowedArgumentCount argumentCount = const AllowedArgumentCount.any(), + bool deterministic = false, + bool directOnly = true}) { + _db.createFunction(functionName: functionName, function: function); + } + + @override + void dispose() { + _db.dispose(); + } + + @override + void execute(String sql, [List parameters = const []]) { + _db.execute(sql, parameters); + } + + @override + int getUpdatedRows() { + // ignore: deprecated_member_use + return _db.getUpdatedRows(); + } + + @override + int get lastInsertRowId => _db.lastInsertRowId; + + @override + CommonPreparedStatement prepare(String sql, + {bool persistent = false, bool vtab = true, bool checkNoTail = false}) { + return _db.prepare(sql, + persistent: persistent, vtab: vtab, checkNoTail: checkNoTail); + } + + @override + List prepareMultiple(String sql, + {bool persistent = false, bool vtab = true}) { + return _db.prepareMultiple(sql, persistent: persistent, vtab: vtab); + } + + @override + ResultSet select(String sql, [List parameters = const []]) { + bool preAutocommit = _db.autocommit; + final result = _db.select(sql, parameters); + bool postAutocommit = _db.autocommit; + if (!preAutocommit && postAutocommit) { + _transactionController.add(true); + } + return result; + } + + @override + int get updatedRows => _db.updatedRows; + + @override + Stream get updates { + return throttledUpdates(_db, _transactionController.stream); + } +} + +/// This throttles the database update stream to: +/// 1. Trigger max once every 1ms. +/// 2. Only trigger _after_ transactions. +Stream throttledUpdates( + CommonDatabase source, Stream transactionStream) { + StreamController? controller; + Set insertedTables = {}; + Set updatedTables = {}; + Set deletedTables = {}; + var paused = false; + + Timer? updateDebouncer; + + void maybeFireUpdates() { + updateDebouncer?.cancel(); + updateDebouncer = null; + + if (paused) { + // Continue collecting updates, but don't fire any + return; + } + + if (!source.autocommit) { + // Inside a transaction - do not fire updates + return; + } + + if (updatedTables.isNotEmpty) { + for (var tableName in updatedTables) { + controller!.add(SqliteUpdate(SqliteUpdateKind.update, tableName, 0)); + } + + updatedTables.clear(); + } + + if (insertedTables.isNotEmpty) { + for (var tableName in insertedTables) { + controller!.add(SqliteUpdate(SqliteUpdateKind.insert, tableName, 0)); + } + + insertedTables.clear(); + } + + if (deletedTables.isNotEmpty) { + for (var tableName in deletedTables) { + controller!.add(SqliteUpdate(SqliteUpdateKind.delete, tableName, 0)); + } + + deletedTables.clear(); + } + } + + void collectUpdate(SqliteUpdate event) { + if (event.kind == SqliteUpdateKind.insert) { + insertedTables.add(event.tableName); + } else if (event.kind == SqliteUpdateKind.update) { + updatedTables.add(event.tableName); + } else if (event.kind == SqliteUpdateKind.delete) { + deletedTables.add(event.tableName); + } + + updateDebouncer ??= + Timer(const Duration(milliseconds: 1), maybeFireUpdates); + } + + StreamSubscription? txSubscription; + StreamSubscription? sourceSubscription; + + controller = StreamController(onListen: () { + txSubscription = transactionStream.listen((event) { + maybeFireUpdates(); + }, onError: (error) { + controller?.addError(error); + }); + + sourceSubscription = source.updates.listen(collectUpdate, onError: (error) { + controller?.addError(error); + }); + }, onPause: () { + paused = true; + }, onResume: () { + paused = false; + maybeFireUpdates(); + }, onCancel: () { + txSubscription?.cancel(); + sourceSubscription?.cancel(); + }); + + return controller.stream; +} diff --git a/packages/sqlite_async/lib/src/web/worker/worker_utils.dart b/packages/sqlite_async/lib/src/web/worker/worker_utils.dart index b4657dd..1d8fb5c 100644 --- a/packages/sqlite_async/lib/src/web/worker/worker_utils.dart +++ b/packages/sqlite_async/lib/src/web/worker/worker_utils.dart @@ -4,6 +4,7 @@ import 'dart:js_util' as js_util; import 'package:mutex/mutex.dart'; import 'package:sqlite3/wasm.dart'; import 'package:sqlite3_web/sqlite3_web.dart'; +import 'throttled_common_database.dart'; import '../protocol.dart'; @@ -18,7 +19,9 @@ base class AsyncSqliteController extends DatabaseController { // Register any custom functions here if needed - return AsyncSqliteDatabase(database: db); + final throttled = ThrottledCommonDatabase(db); + + return AsyncSqliteDatabase(database: throttled); } @override diff --git a/packages/sqlite_async/test/watch_test.dart b/packages/sqlite_async/test/watch_test.dart index b179009..08a80cb 100644 --- a/packages/sqlite_async/test/watch_test.dart +++ b/packages/sqlite_async/test/watch_test.dart @@ -258,7 +258,7 @@ void main() { final db = await testUtils.setupDatabase(path: path); await createTables(db); - const baseTime = 20; + const baseTime = 10; const throttleDuration = Duration(milliseconds: baseTime); // delay must be bigger than throttleDuration, and bigger @@ -293,6 +293,13 @@ void main() { // one event after the transaction 2 ])); + + // Other observed results (failure scenarios): + // [0, 0, 0]: The watch is triggered during the transaction + // and executes concurrently with the transaction. + // [0, 2, 2]: The watch is triggered during the transaction, + // but executes after the transaction (single connection). + // [0]: No updates triggered. }); }); } From deaaba52c1f30c91fc67129dc513b4278f5fbe58 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Thu, 17 Oct 2024 16:17:53 +0200 Subject: [PATCH 15/90] Simplify update throttling. --- .../web/worker/throttled_common_database.dart | 38 ++++--------------- 1 file changed, 8 insertions(+), 30 deletions(-) diff --git a/packages/sqlite_async/lib/src/web/worker/throttled_common_database.dart b/packages/sqlite_async/lib/src/web/worker/throttled_common_database.dart index da69d12..ea73bd6 100644 --- a/packages/sqlite_async/lib/src/web/worker/throttled_common_database.dart +++ b/packages/sqlite_async/lib/src/web/worker/throttled_common_database.dart @@ -111,9 +111,7 @@ class ThrottledCommonDatabase extends CommonDatabase { Stream throttledUpdates( CommonDatabase source, Stream transactionStream) { StreamController? controller; - Set insertedTables = {}; - Set updatedTables = {}; - Set deletedTables = {}; + Set pendingUpdates = {}; var paused = false; Timer? updateDebouncer; @@ -132,39 +130,19 @@ Stream throttledUpdates( return; } - if (updatedTables.isNotEmpty) { - for (var tableName in updatedTables) { - controller!.add(SqliteUpdate(SqliteUpdateKind.update, tableName, 0)); + if (pendingUpdates.isNotEmpty) { + for (var update in pendingUpdates) { + controller!.add(update); } - updatedTables.clear(); - } - - if (insertedTables.isNotEmpty) { - for (var tableName in insertedTables) { - controller!.add(SqliteUpdate(SqliteUpdateKind.insert, tableName, 0)); - } - - insertedTables.clear(); - } - - if (deletedTables.isNotEmpty) { - for (var tableName in deletedTables) { - controller!.add(SqliteUpdate(SqliteUpdateKind.delete, tableName, 0)); - } - - deletedTables.clear(); + pendingUpdates.clear(); } } void collectUpdate(SqliteUpdate event) { - if (event.kind == SqliteUpdateKind.insert) { - insertedTables.add(event.tableName); - } else if (event.kind == SqliteUpdateKind.update) { - updatedTables.add(event.tableName); - } else if (event.kind == SqliteUpdateKind.delete) { - deletedTables.add(event.tableName); - } + // We merge updates with the same kind and tableName. + // rowId is never used in sqlite_async. + pendingUpdates.add(SqliteUpdate(event.kind, event.tableName, 0)); updateDebouncer ??= Timer(const Duration(milliseconds: 1), maybeFireUpdates); From c36a1386d3547b374ae127dce7d5571087fd7560 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Thu, 17 Oct 2024 17:00:01 +0200 Subject: [PATCH 16/90] Make drift tests stable. --- packages/drift_sqlite_async/test/db_test.dart | 31 +++++++++++++------ 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/packages/drift_sqlite_async/test/db_test.dart b/packages/drift_sqlite_async/test/db_test.dart index ed901cf..6de575e 100644 --- a/packages/drift_sqlite_async/test/db_test.dart +++ b/packages/drift_sqlite_async/test/db_test.dart @@ -48,8 +48,15 @@ void main() { test('watch', () async { var stream = dbu.select(dbu.todoItems).watch(); - var resultsPromise = - stream.distinct().skipWhile((e) => e.isEmpty).take(3).toList(); + var resultsPromise = stream + // toString() so that we can use distinct() + .map((rows) => rows.toString()) + // Drift may or may not emit duplicate update notifications. + // We use distinct() to ignore those. + .distinct() + .skipWhile((e) => e.isEmpty) + .take(3) + .toList(); await dbu.into(dbu.todoItems).insert( TodoItemsCompanion.insert(id: Value(1), description: 'Test 1')); @@ -65,16 +72,20 @@ void main() { expect( results, equals([ - [TodoItem(id: 1, description: 'Test 1')], - [TodoItem(id: 1, description: 'Test 1B')], - [] + '[TodoItem(id: 1, description: Test 1)]', + '[TodoItem(id: 1, description: Test 1B)]', + '[]' ])); }); test('watch with external updates', () async { var stream = dbu.select(dbu.todoItems).watch(); - var resultsPromise = - stream.distinct().skipWhile((e) => e.isEmpty).take(3).toList(); + var resultsPromise = stream + .map((rows) => rows.toString()) + .distinct() + .skipWhile((e) => e.isEmpty) + .take(3) + .toList(); await db.execute( 'INSERT INTO todos(id, description) VALUES(?, ?)', [1, 'Test 1']); @@ -88,9 +99,9 @@ void main() { expect( results, equals([ - [TodoItem(id: 1, description: 'Test 1')], - [TodoItem(id: 1, description: 'Test 1B')], - [] + '[TodoItem(id: 1, description: Test 1)]', + '[TodoItem(id: 1, description: Test 1B)]', + '[]' ])); }); }); From 8b40681828fc81924a3e4de936696968a40966f5 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Mon, 28 Oct 2024 11:38:32 +0200 Subject: [PATCH 17/90] v0.9.1; bump dependencies. --- packages/sqlite_async/CHANGELOG.md | 12 +++++++++--- packages/sqlite_async/pubspec.yaml | 6 +++--- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/packages/sqlite_async/CHANGELOG.md b/packages/sqlite_async/CHANGELOG.md index a91538e..f1dad95 100644 --- a/packages/sqlite_async/CHANGELOG.md +++ b/packages/sqlite_async/CHANGELOG.md @@ -1,12 +1,18 @@ +## 0.9.1 + +- Support version ^0.2.0 of package:sqlite3_web +- Fix update notifications to only fire outside transactions +- Fix update notifications to be debounced on web + ## 0.9.0 - - Support the latest version of package:web and package:sqlite3_web +- Support the latest version of package:web and package:sqlite3_web - - Export sqlite3 `open` for packages that depend on `sqlite_async` +- Export sqlite3 `open` for packages that depend on `sqlite_async` ## 0.8.3 - - Updated web database implementation for get and getOptional. Fixed refreshSchema not working in web. +- Updated web database implementation for get and getOptional. Fixed refreshSchema not working in web. ## 0.8.2 diff --git a/packages/sqlite_async/pubspec.yaml b/packages/sqlite_async/pubspec.yaml index 438acbb..c412233 100644 --- a/packages/sqlite_async/pubspec.yaml +++ b/packages/sqlite_async/pubspec.yaml @@ -1,6 +1,6 @@ name: sqlite_async description: High-performance asynchronous interface for SQLite on Dart and Flutter. -version: 0.9.0 +version: 0.9.1 repository: https://github.com/powersync-ja/sqlite_async.dart environment: sdk: ">=3.4.0 <4.0.0" @@ -12,8 +12,8 @@ topics: - flutter dependencies: - sqlite3: "^2.4.4" - sqlite3_web: ^0.1.3 + sqlite3: "^2.4.7" + sqlite3_web: ^0.2.0 async: ^2.10.0 collection: ^1.17.0 mutex: ^3.1.0 From c39e47544a2d8114aba69947cb73e962b84a9850 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Mon, 28 Oct 2024 11:47:16 +0200 Subject: [PATCH 18/90] Improve test stability. --- .github/workflows/test.yaml | 1 + packages/sqlite_async/test/watch_test.dart | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 85c9c02..2cd34e0 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -30,6 +30,7 @@ jobs: test: runs-on: ubuntu-latest strategy: + fail-fast: false matrix: include: - sqlite_version: "3440200" diff --git a/packages/sqlite_async/test/watch_test.dart b/packages/sqlite_async/test/watch_test.dart index 08a80cb..7e74790 100644 --- a/packages/sqlite_async/test/watch_test.dart +++ b/packages/sqlite_async/test/watch_test.dart @@ -258,7 +258,7 @@ void main() { final db = await testUtils.setupDatabase(path: path); await createTables(db); - const baseTime = 10; + const baseTime = 20; const throttleDuration = Duration(milliseconds: baseTime); // delay must be bigger than throttleDuration, and bigger @@ -300,6 +300,7 @@ void main() { // [0, 2, 2]: The watch is triggered during the transaction, // but executes after the transaction (single connection). // [0]: No updates triggered. + // [2, 2]: Timing issue? }); }); } From 490e5c3ac7f208eedc8d07e790dd12a799d85e79 Mon Sep 17 00:00:00 2001 From: Mughees Khan Date: Mon, 28 Oct 2024 18:14:13 +0200 Subject: [PATCH 19/90] Feat: Expose worker connection (#72) * Add methods for exchanging database connections * Expose lock name as well * Also implement interface in sqlite database impl * Add changelog entry * Web: Return when database is closed --------- Co-authored-by: Simon Binder --- packages/sqlite_async/CHANGELOG.md | 6 ++ .../sqlite_async/lib/src/web/database.dart | 21 +++++- .../src/web/database/web_sqlite_database.dart | 19 ++++-- .../sqlite_async/lib/src/web/web_mutex.dart | 6 +- packages/sqlite_async/lib/web.dart | 68 +++++++++++++++++++ packages/sqlite_async/pubspec.yaml | 2 +- 6 files changed, 113 insertions(+), 9 deletions(-) create mode 100644 packages/sqlite_async/lib/web.dart diff --git a/packages/sqlite_async/CHANGELOG.md b/packages/sqlite_async/CHANGELOG.md index f1dad95..93d8166 100644 --- a/packages/sqlite_async/CHANGELOG.md +++ b/packages/sqlite_async/CHANGELOG.md @@ -1,3 +1,9 @@ +## 0.10.0 + +- Add the `exposeEndpoint()` method available on web databases. It returns a serializable + description of the database endpoint that can be sent across workers. + This allows sharing an opened database connection across workers. + ## 0.9.1 - Support version ^0.2.0 of package:sqlite3_web diff --git a/packages/sqlite_async/lib/src/web/database.dart b/packages/sqlite_async/lib/src/web/database.dart index b632aa7..96026d2 100644 --- a/packages/sqlite_async/lib/src/web/database.dart +++ b/packages/sqlite_async/lib/src/web/database.dart @@ -5,11 +5,13 @@ import 'package:sqlite3/common.dart'; import 'package:sqlite3_web/sqlite3_web.dart'; import 'package:sqlite_async/sqlite_async.dart'; import 'package:sqlite_async/src/utils/shared_utils.dart'; +import 'package:sqlite_async/web.dart'; import 'protocol.dart'; +import 'web_mutex.dart'; class WebDatabase with SqliteQueries, SqliteDatabaseMixin - implements SqliteDatabase { + implements SqliteDatabase, WebSqliteConnection { final Database _database; final Mutex? _mutex; @@ -24,6 +26,9 @@ class WebDatabase closed = true; } + @override + Future get closedFuture => _database.closed; + @override Future getAutoCommit() async { final response = await _database.customRequest( @@ -56,6 +61,20 @@ class WebDatabase /// Not relevant for web. Never get openFactory => throw UnimplementedError(); + @override + Future exposeEndpoint() async { + final endpoint = await _database.additionalConnection(); + + return ( + connectPort: endpoint.$1, + connectName: endpoint.$2, + lockName: switch (_mutex) { + MutexImpl(:final resolvedIdentifier) => resolvedIdentifier, + _ => null, + }, + ); + } + @override Future readLock(Future Function(SqliteReadContext tx) callback, {Duration? lockTimeout, String? debugContext}) async { diff --git a/packages/sqlite_async/lib/src/web/database/web_sqlite_database.dart b/packages/sqlite_async/lib/src/web/database/web_sqlite_database.dart index 522b48e..c7e2a70 100644 --- a/packages/sqlite_async/lib/src/web/database/web_sqlite_database.dart +++ b/packages/sqlite_async/lib/src/web/database/web_sqlite_database.dart @@ -10,17 +10,23 @@ import 'package:sqlite_async/src/sqlite_options.dart'; import 'package:sqlite_async/src/update_notification.dart'; import 'package:sqlite_async/src/web/web_mutex.dart'; import 'package:sqlite_async/src/web/web_sqlite_open_factory.dart'; +import 'package:sqlite_async/web.dart'; + +import '../database.dart'; /// Web implementation of [SqliteDatabase] /// Uses a web worker for SQLite connection class SqliteDatabaseImpl with SqliteQueries, SqliteDatabaseMixin - implements SqliteDatabase { + implements SqliteDatabase, WebSqliteConnection { @override bool get closed { return _connection.closed; } + @override + Future get closedFuture => _connection.closedFuture; + final StreamController updatesController = StreamController.broadcast(); @@ -38,7 +44,7 @@ class SqliteDatabaseImpl AbstractDefaultSqliteOpenFactory openFactory; late final Mutex mutex; - late final SqliteConnection _connection; + late final WebDatabase _connection; /// Open a SqliteDatabase. /// @@ -78,8 +84,8 @@ class SqliteDatabaseImpl Future _init() async { _connection = await openFactory.openConnection(SqliteOpenOptions( - primaryConnection: true, readOnly: false, mutex: mutex)); - _connection.updates!.forEach((update) { + primaryConnection: true, readOnly: false, mutex: mutex)) as WebDatabase; + _connection.updates.forEach((update) { updatesController.add(update); }); } @@ -139,4 +145,9 @@ class SqliteDatabaseImpl await isInitialized; return _connection.getAutoCommit(); } + + @override + Future exposeEndpoint() async { + return await _connection.exposeEndpoint(); + } } diff --git a/packages/sqlite_async/lib/src/web/web_mutex.dart b/packages/sqlite_async/lib/src/web/web_mutex.dart index 4013201..38f7ff5 100644 --- a/packages/sqlite_async/lib/src/web/web_mutex.dart +++ b/packages/sqlite_async/lib/src/web/web_mutex.dart @@ -18,7 +18,7 @@ external Navigator get _navigator; class MutexImpl implements Mutex { late final mutex.Mutex fallback; String? identifier; - final String _resolvedIdentifier; + final String resolvedIdentifier; MutexImpl({this.identifier}) @@ -29,7 +29,7 @@ class MutexImpl implements Mutex { /// - The uuid package could be added for better uniqueness if required. /// This would add another package dependency to `sqlite_async` which is potentially unnecessary at this point. /// An identifier should be supplied for better exclusion. - : _resolvedIdentifier = identifier ?? + : resolvedIdentifier = identifier ?? "${DateTime.now().microsecondsSinceEpoch}-${Random().nextDouble()}" { fallback = mutex.Mutex(); } @@ -125,7 +125,7 @@ class MutexImpl implements Mutex { final lockOptions = JSObject(); lockOptions['signal'] = controller.signal; final promise = _navigator.locks - .request(_resolvedIdentifier, lockOptions, jsCallback.toJS); + .request(resolvedIdentifier, lockOptions, jsCallback.toJS); // A timeout abort will throw an exception which needs to be handled. // There should not be any other unhandled lock errors. js_util.promiseToFuture(promise).catchError((error) {}); diff --git a/packages/sqlite_async/lib/web.dart b/packages/sqlite_async/lib/web.dart new file mode 100644 index 0000000..7c49737 --- /dev/null +++ b/packages/sqlite_async/lib/web.dart @@ -0,0 +1,68 @@ +/// Exposes interfaces implemented by database implementations on the web. +/// +/// These expose methods allowing database instances to be shared across web +/// workers. +library sqlite_async.web; + +import 'package:sqlite3_web/sqlite3_web.dart'; +import 'package:web/web.dart'; +import 'sqlite_async.dart'; +import 'src/web/database.dart'; + +/// An endpoint that can be used, by any running JavaScript context in the same +/// website, to connect to an existing [WebSqliteConnection]. +/// +/// These endpoints are created by calling [WebSqliteConnection.exposeEndpoint] +/// and consist of a [MessagePort] and two [String]s internally identifying the +/// connection. Both objects can be transferred over send ports towards another +/// worker or context. That context can then use +/// [WebSqliteConnection.connectToEndpoint] to connect to the port already +/// opened. +typedef WebDatabaseEndpoint = ({ + MessagePort connectPort, + String connectName, + String? lockName, +}); + +/// A [SqliteConnection] interface implemented by opened connections when +/// running on the web. +/// +/// This adds the [exposeEndpoint], which uses `dart:js_interop` types not +/// supported on native Dart platforms. The method can be used to access an +/// opened database across different JavaScript contexts +/// (e.g. document windows and workers). +abstract class WebSqliteConnection implements SqliteConnection { + /// Returns a future that completes when this connection is closed. + /// + /// This usually only happens when calling [close], but on the web + /// specifically, it can also happen when a remote context closes a database + /// accessed via [connectToEndpoint]. + Future get closedFuture; + + /// Returns a [WebDatabaseEndpoint] - a structure that consists only of types + /// that can be transferred across a [MessagePort] in JavaScript. + /// + /// After transferring this endpoint to another JavaScript context (e.g. a + /// worker), the worker can call [connectToEndpoint] to obtain a connection to + /// the same sqlite database. + Future exposeEndpoint(); + + /// Connect to an endpoint obtained through [exposeEndpoint]. + /// + /// The endpoint is transferrable in JavaScript, allowing multiple JavaScript + /// contexts to exchange opened database connections. + static Future connectToEndpoint( + WebDatabaseEndpoint endpoint) async { + final rawSqlite = await WebSqlite.connectToPort( + (endpoint.connectPort, endpoint.connectName)); + + final database = WebDatabase( + rawSqlite, + switch (endpoint.lockName) { + var lock? => Mutex(identifier: lock), + null => null, + }, + ); + return database; + } +} diff --git a/packages/sqlite_async/pubspec.yaml b/packages/sqlite_async/pubspec.yaml index c412233..b6ab802 100644 --- a/packages/sqlite_async/pubspec.yaml +++ b/packages/sqlite_async/pubspec.yaml @@ -1,6 +1,6 @@ name: sqlite_async description: High-performance asynchronous interface for SQLite on Dart and Flutter. -version: 0.9.1 +version: 0.10.0 repository: https://github.com/powersync-ja/sqlite_async.dart environment: sdk: ">=3.4.0 <4.0.0" From a62c9e80acefc90aebcca5d22fb825fc787ab9f8 Mon Sep 17 00:00:00 2001 From: Mughees Khan Date: Mon, 28 Oct 2024 19:50:32 +0200 Subject: [PATCH 20/90] Fix: drift_sqlite_async version (#73) * Bump sqlite_async to v0.10.0 * chore(release): publish packages - drift_sqlite_async@0.2.0-alpha.2 * Fix sqlite_async version --- CHANGELOG.md | 28 ++++++++++++++++++++++++ packages/drift_sqlite_async/CHANGELOG.md | 4 ++++ packages/drift_sqlite_async/pubspec.yaml | 4 ++-- 3 files changed, 34 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 19e76bf..4a1ed02 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,34 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## 2024-10-28 + +### Changes + +--- + +Packages with breaking changes: + + - There are no breaking changes in this release. + +Packages with other changes: + + - [`drift_sqlite_async` - `v0.2.0-alpha.2`](#drift_sqlite_async---v020-alpha2) + - [`sqlite_async` - `v0.10.0`](#sqlite_async---v0100) + +--- + +#### `drift_sqlite_async` - `v0.2.0-alpha.2` + + - Bump sqlite_async to v0.10.0 + +#### `sqlite_async` - `v0.10.0` + + - Add the `exposeEndpoint()` method available on web databases. It returns a serializable + description of the database endpoint that can be sent across workers. + This allows sharing an opened database connection across workers. + + ## 2024-09-03 ### Changes diff --git a/packages/drift_sqlite_async/CHANGELOG.md b/packages/drift_sqlite_async/CHANGELOG.md index c82b6d7..0999642 100644 --- a/packages/drift_sqlite_async/CHANGELOG.md +++ b/packages/drift_sqlite_async/CHANGELOG.md @@ -1,3 +1,7 @@ +## 0.2.0-alpha.2 + + - Bump `sqlite_async` to v0.10.0 + ## 0.2.0-alpha.1 - Support `drift` version >=2.19 and `web` v1.0.0. diff --git a/packages/drift_sqlite_async/pubspec.yaml b/packages/drift_sqlite_async/pubspec.yaml index b168037..ad2622e 100644 --- a/packages/drift_sqlite_async/pubspec.yaml +++ b/packages/drift_sqlite_async/pubspec.yaml @@ -1,5 +1,5 @@ name: drift_sqlite_async -version: 0.2.0-alpha.1 +version: 0.2.0-alpha.2 homepage: https://github.com/powersync-ja/sqlite_async.dart repository: https://github.com/powersync-ja/sqlite_async.dart description: Use Drift with a sqlite_async database, allowing both to be used in the same application. @@ -15,7 +15,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: drift: ">=2.19.0 <3.0.0" - sqlite_async: ^0.9.0 + sqlite_async: ^0.10.0 dev_dependencies: build_runner: ^2.4.8 drift_dev: ">=2.19.0 <3.0.0" From e26fb09d2d39216491d231cffe24e9735db88a49 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Fri, 1 Nov 2024 09:48:36 +0100 Subject: [PATCH 21/90] Broadcast updates across tabs if needed (#74) * Share database updates across tabs * Avoid unecessary toList() * Remove unused constructor * Also use events locally * Explain duplicate add --- .../sqlite_async/lib/src/web/database.dart | 7 ++- .../src/web/database/broadcast_updates.dart | 50 +++++++++++++++++++ .../src/web/database/web_sqlite_database.dart | 28 +++++++++-- .../lib/src/web/web_sqlite_open_factory.dart | 10 +++- 4 files changed, 90 insertions(+), 5 deletions(-) create mode 100644 packages/sqlite_async/lib/src/web/database/broadcast_updates.dart diff --git a/packages/sqlite_async/lib/src/web/database.dart b/packages/sqlite_async/lib/src/web/database.dart index 96026d2..3e3233b 100644 --- a/packages/sqlite_async/lib/src/web/database.dart +++ b/packages/sqlite_async/lib/src/web/database.dart @@ -5,6 +5,7 @@ import 'package:sqlite3/common.dart'; import 'package:sqlite3_web/sqlite3_web.dart'; import 'package:sqlite_async/sqlite_async.dart'; import 'package:sqlite_async/src/utils/shared_utils.dart'; +import 'package:sqlite_async/src/web/database/broadcast_updates.dart'; import 'package:sqlite_async/web.dart'; import 'protocol.dart'; import 'web_mutex.dart'; @@ -15,10 +16,14 @@ class WebDatabase final Database _database; final Mutex? _mutex; + /// For persistent databases that aren't backed by a shared worker, we use + /// web broadcast channels to forward local update events to other tabs. + final BroadcastUpdates? broadcastUpdates; + @override bool closed = false; - WebDatabase(this._database, this._mutex); + WebDatabase(this._database, this._mutex, {this.broadcastUpdates}); @override Future close() async { diff --git a/packages/sqlite_async/lib/src/web/database/broadcast_updates.dart b/packages/sqlite_async/lib/src/web/database/broadcast_updates.dart new file mode 100644 index 0000000..5514e74 --- /dev/null +++ b/packages/sqlite_async/lib/src/web/database/broadcast_updates.dart @@ -0,0 +1,50 @@ +import 'dart:js_interop'; + +import 'package:sqlite_async/sqlite_async.dart'; +import 'package:web/web.dart' as web; + +/// Utility to share received [UpdateNotification]s with other tabs using +/// broadcast channels. +class BroadcastUpdates { + final web.BroadcastChannel _channel; + + BroadcastUpdates(String name) + : _channel = web.BroadcastChannel('sqlite3_async_updates/$name'); + + Stream get updates { + return web.EventStreamProviders.messageEvent + .forTarget(_channel) + .map((event) { + final data = event.data as _BroadcastMessage; + if (data.a == 0) { + final payload = data.b as JSArray; + return UpdateNotification( + payload.toDart.map((e) => e.toDart).toSet()); + } else { + return null; + } + }) + .where((e) => e != null) + .cast(); + } + + void send(UpdateNotification notification) { + _channel.postMessage(_BroadcastMessage.notifications(notification)); + } +} + +@JS() +@anonymous +extension type _BroadcastMessage._(JSObject _) implements JSObject { + external int get a; + external JSAny get b; + + external factory _BroadcastMessage({required int a, required JSAny b}); + + factory _BroadcastMessage.notifications(UpdateNotification notification) { + return _BroadcastMessage( + a: 0, + b: notification.tables.map((e) => e.toJS).toList().toJS, + ); + } +} diff --git a/packages/sqlite_async/lib/src/web/database/web_sqlite_database.dart b/packages/sqlite_async/lib/src/web/database/web_sqlite_database.dart index c7e2a70..c00377c 100644 --- a/packages/sqlite_async/lib/src/web/database/web_sqlite_database.dart +++ b/packages/sqlite_async/lib/src/web/database/web_sqlite_database.dart @@ -45,6 +45,7 @@ class SqliteDatabaseImpl late final Mutex mutex; late final WebDatabase _connection; + StreamSubscription? _broadcastUpdatesSubscription; /// Open a SqliteDatabase. /// @@ -85,9 +86,28 @@ class SqliteDatabaseImpl Future _init() async { _connection = await openFactory.openConnection(SqliteOpenOptions( primaryConnection: true, readOnly: false, mutex: mutex)) as WebDatabase; - _connection.updates.forEach((update) { - updatesController.add(update); - }); + + final broadcastUpdates = _connection.broadcastUpdates; + if (broadcastUpdates == null) { + // We can use updates directly from the database. + _connection.updates.forEach((update) { + updatesController.add(update); + }); + } else { + _connection.updates.forEach((update) { + updatesController.add(update); + + // Share local updates with other tabs + broadcastUpdates.send(update); + }); + + // Also add updates from other tabs, note that things we send aren't + // received by our tab. + _broadcastUpdatesSubscription = + broadcastUpdates.updates.listen((updates) { + updatesController.add(updates); + }); + } } T _runZoned(T Function() callback, {required String debugContext}) { @@ -132,6 +152,8 @@ class SqliteDatabaseImpl @override Future close() async { await isInitialized; + _broadcastUpdatesSubscription?.cancel(); + updatesController.close(); return _connection.close(); } diff --git a/packages/sqlite_async/lib/src/web/web_sqlite_open_factory.dart b/packages/sqlite_async/lib/src/web/web_sqlite_open_factory.dart index 521320b..66000f9 100644 --- a/packages/sqlite_async/lib/src/web/web_sqlite_open_factory.dart +++ b/packages/sqlite_async/lib/src/web/web_sqlite_open_factory.dart @@ -3,6 +3,7 @@ import 'dart:async'; import 'package:sqlite3/wasm.dart'; import 'package:sqlite3_web/sqlite3_web.dart'; import 'package:sqlite_async/sqlite_async.dart'; +import 'package:sqlite_async/src/web/database/broadcast_updates.dart'; import 'package:sqlite_async/src/web/web_mutex.dart'; import 'database.dart'; @@ -57,7 +58,14 @@ class DefaultSqliteOpenFactory ? null : MutexImpl(identifier: path); // Use the DB path as a mutex identifier - return WebDatabase(connection.database, options.mutex ?? mutex); + BroadcastUpdates? updates; + if (connection.access != AccessMode.throughSharedWorker && + connection.storage != StorageMode.inMemory) { + updates = BroadcastUpdates(path); + } + + return WebDatabase(connection.database, options.mutex ?? mutex, + broadcastUpdates: updates); } @override From 3cdd5dd03f7cd32b5603f410eeb599148d4cfbd0 Mon Sep 17 00:00:00 2001 From: Mughees Khan Date: Fri, 1 Nov 2024 11:18:06 +0200 Subject: [PATCH 22/90] chore(release): publish packages (#75) - sqlite_async@0.10.1 - drift_sqlite_async@0.2.0-alpha.3 --- CHANGELOG.md | 26 ++++++++++++++++++++++++ packages/drift_sqlite_async/CHANGELOG.md | 4 ++++ packages/drift_sqlite_async/pubspec.yaml | 4 ++-- packages/sqlite_async/CHANGELOG.md | 4 ++++ packages/sqlite_async/pubspec.yaml | 2 +- 5 files changed, 37 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a1ed02..b9886c5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,32 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## 2024-11-01 + +### Changes + +--- + +Packages with breaking changes: + + - There are no breaking changes in this release. + +Packages with other changes: + + - [`sqlite_async` - `v0.10.1`](#sqlite_async---v0101) + - [`drift_sqlite_async` - `v0.2.0-alpha.3`](#drift_sqlite_async---v020-alpha3) + +--- + +#### `sqlite_async` - `v0.10.1` + + - For database setups not using a shared worker, use a `BroadcastChannel` to share updates across different tabs. + +#### `drift_sqlite_async` - `v0.2.0-alpha.3` + + - Bump `sqlite_async` to v0.10.1 + + ## 2024-10-28 ### Changes diff --git a/packages/drift_sqlite_async/CHANGELOG.md b/packages/drift_sqlite_async/CHANGELOG.md index 0999642..de46be7 100644 --- a/packages/drift_sqlite_async/CHANGELOG.md +++ b/packages/drift_sqlite_async/CHANGELOG.md @@ -1,3 +1,7 @@ +## 0.2.0-alpha.3 + + - Bump `sqlite_async` to v0.10.1 + ## 0.2.0-alpha.2 - Bump `sqlite_async` to v0.10.0 diff --git a/packages/drift_sqlite_async/pubspec.yaml b/packages/drift_sqlite_async/pubspec.yaml index ad2622e..109314d 100644 --- a/packages/drift_sqlite_async/pubspec.yaml +++ b/packages/drift_sqlite_async/pubspec.yaml @@ -1,5 +1,5 @@ name: drift_sqlite_async -version: 0.2.0-alpha.2 +version: 0.2.0-alpha.3 homepage: https://github.com/powersync-ja/sqlite_async.dart repository: https://github.com/powersync-ja/sqlite_async.dart description: Use Drift with a sqlite_async database, allowing both to be used in the same application. @@ -15,7 +15,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: drift: ">=2.19.0 <3.0.0" - sqlite_async: ^0.10.0 + sqlite_async: ^0.10.1 dev_dependencies: build_runner: ^2.4.8 drift_dev: ">=2.19.0 <3.0.0" diff --git a/packages/sqlite_async/CHANGELOG.md b/packages/sqlite_async/CHANGELOG.md index 93d8166..88f9875 100644 --- a/packages/sqlite_async/CHANGELOG.md +++ b/packages/sqlite_async/CHANGELOG.md @@ -1,3 +1,7 @@ +## 0.10.1 + +- For database setups not using a shared worker, use a `BroadcastChannel` to share updates across different tabs. + ## 0.10.0 - Add the `exposeEndpoint()` method available on web databases. It returns a serializable diff --git a/packages/sqlite_async/pubspec.yaml b/packages/sqlite_async/pubspec.yaml index b6ab802..453f154 100644 --- a/packages/sqlite_async/pubspec.yaml +++ b/packages/sqlite_async/pubspec.yaml @@ -1,6 +1,6 @@ name: sqlite_async description: High-performance asynchronous interface for SQLite on Dart and Flutter. -version: 0.10.0 +version: 0.10.1 repository: https://github.com/powersync-ja/sqlite_async.dart environment: sdk: ">=3.4.0 <4.0.0" From ed2c3c035d8189179a14c929686a64df877b8dcf Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Mon, 4 Nov 2024 15:43:02 +0200 Subject: [PATCH 23/90] Flush at the end of each writeLock. --- packages/sqlite_async/lib/src/web/database.dart | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/sqlite_async/lib/src/web/database.dart b/packages/sqlite_async/lib/src/web/database.dart index 3e3233b..f392684 100644 --- a/packages/sqlite_async/lib/src/web/database.dart +++ b/packages/sqlite_async/lib/src/web/database.dart @@ -137,6 +137,7 @@ class WebDatabase return await callback(context); } finally { context.markClosed(); + await _database.fileSystem.flush(); } }); } else { @@ -148,6 +149,7 @@ class WebDatabase return await callback(context); } finally { context.markClosed(); + await _database.fileSystem.flush(); await _database.customRequest( CustomDatabaseMessage(CustomDatabaseMessageKind.releaseLock)); } From c06e4ec6377a2e9df6297e9dc7d46bcb039f97f1 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 5 Nov 2024 14:23:09 +0200 Subject: [PATCH 24/90] Make flush optional. --- .../sqlite_async/lib/src/web/database.dart | 22 ++++++++++---- .../src/web/database/web_sqlite_database.dart | 16 +++++++--- packages/sqlite_async/lib/web.dart | 30 +++++++++++++++++++ 3 files changed, 59 insertions(+), 9 deletions(-) diff --git a/packages/sqlite_async/lib/src/web/database.dart b/packages/sqlite_async/lib/src/web/database.dart index f392684..a4f0ddf 100644 --- a/packages/sqlite_async/lib/src/web/database.dart +++ b/packages/sqlite_async/lib/src/web/database.dart @@ -113,7 +113,8 @@ class WebDatabase @override Future writeTransaction( Future Function(SqliteWriteContext tx) callback, - {Duration? lockTimeout}) { + {Duration? lockTimeout, + bool? flush}) { return writeLock( (writeContext) => internalWriteTransaction(writeContext, (context) async { @@ -122,14 +123,15 @@ class WebDatabase return callback(_ExclusiveTransactionContext(this, writeContext)); }), debugContext: 'writeTransaction()', - lockTimeout: lockTimeout); + lockTimeout: lockTimeout, + flush: flush); } @override /// Internal writeLock which intercepts transaction context's to verify auto commit is not active Future writeLock(Future Function(SqliteWriteContext tx) callback, - {Duration? lockTimeout, String? debugContext}) async { + {Duration? lockTimeout, String? debugContext, bool? flush}) async { if (_mutex case var mutex?) { return await mutex.lock(() async { final context = _ExclusiveContext(this); @@ -137,7 +139,9 @@ class WebDatabase return await callback(context); } finally { context.markClosed(); - await _database.fileSystem.flush(); + if (flush != false) { + await this.flush(); + } } }); } else { @@ -149,12 +153,20 @@ class WebDatabase return await callback(context); } finally { context.markClosed(); - await _database.fileSystem.flush(); + if (flush != false) { + await this.flush(); + } await _database.customRequest( CustomDatabaseMessage(CustomDatabaseMessageKind.releaseLock)); } } } + + @override + Future flush() async { + await isInitialized; + return _database.fileSystem.flush(); + } } class _SharedContext implements SqliteReadContext { diff --git a/packages/sqlite_async/lib/src/web/database/web_sqlite_database.dart b/packages/sqlite_async/lib/src/web/database/web_sqlite_database.dart index c00377c..0f38b1c 100644 --- a/packages/sqlite_async/lib/src/web/database/web_sqlite_database.dart +++ b/packages/sqlite_async/lib/src/web/database/web_sqlite_database.dart @@ -131,24 +131,32 @@ class SqliteDatabaseImpl @override Future writeLock(Future Function(SqliteWriteContext tx) callback, - {Duration? lockTimeout, String? debugContext}) async { + {Duration? lockTimeout, String? debugContext, bool? flush}) async { await isInitialized; return _runZoned(() { return _connection.writeLock(callback, - lockTimeout: lockTimeout, debugContext: debugContext); + lockTimeout: lockTimeout, debugContext: debugContext, flush: flush); }, debugContext: debugContext ?? 'execute()'); } @override Future writeTransaction( Future Function(SqliteWriteContext tx) callback, - {Duration? lockTimeout}) async { + {Duration? lockTimeout, + bool? flush}) async { await isInitialized; return _runZoned( - () => _connection.writeTransaction(callback, lockTimeout: lockTimeout), + () => _connection.writeTransaction(callback, + lockTimeout: lockTimeout, flush: flush), debugContext: 'writeTransaction()'); } + @override + Future flush() async { + await isInitialized; + return _connection.flush(); + } + @override Future close() async { await isInitialized; diff --git a/packages/sqlite_async/lib/web.dart b/packages/sqlite_async/lib/web.dart index 7c49737..a4005e1 100644 --- a/packages/sqlite_async/lib/web.dart +++ b/packages/sqlite_async/lib/web.dart @@ -65,4 +65,34 @@ abstract class WebSqliteConnection implements SqliteConnection { ); return database; } + + /// Same as [SqliteConnection.writeLock]. + /// + /// Has an additional [flush] (defaults to true). This can be set to false + /// to delay flushing changes to the database file, losing durability guarantees. + /// This only has an effect when IndexedDB storage is used. + /// + /// See [flush] for details. + Future writeLock(Future Function(SqliteWriteContext tx) callback, + {Duration? lockTimeout, String? debugContext, bool? flush}); + + /// Same as [SqliteConnection.writeTransaction]. + /// + /// Has an additional [flush] (defaults to true). This can be set to false + /// to delay flushing changes to the database file, losing durability guarantees. + /// This only has an effect when IndexedDB storage is used. + /// + /// See [flush] for details. + Future writeTransaction( + Future Function(SqliteWriteContext tx) callback, + {Duration? lockTimeout, + bool? flush}); + + /// Flush changes to the underlying storage. + /// + /// When this returns, all changes previously written will be persisted + /// to storage. + /// + /// This only has an effect when IndexedDB storage is used. + Future flush(); } From ce0750e603ac59426dbc783ca4f1736bcc5ecbd8 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Wed, 6 Nov 2024 08:24:59 +0200 Subject: [PATCH 25/90] chore(release): publish packages - sqlite_async@0.11.0 - drift_sqlite_async@0.2.0-alpha.4 --- CHANGELOG.md | 28 ++++++++++++++++++++++++ packages/drift_sqlite_async/CHANGELOG.md | 4 ++++ packages/drift_sqlite_async/pubspec.yaml | 4 ++-- packages/sqlite_async/CHANGELOG.md | 4 ++++ packages/sqlite_async/pubspec.yaml | 2 +- 5 files changed, 39 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b9886c5..f5c9c0d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,34 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## 2024-11-06 + +### Changes + +--- + +Packages with breaking changes: + + - There are no breaking changes in this release. + +Packages with other changes: + + - [`sqlite_async` - `v0.11.0`](#sqlite_async---v0110) + - [`drift_sqlite_async` - `v0.2.0-alpha.4`](#drift_sqlite_async---v020-alpha4) + +Packages with dependency updates only: + +> Packages listed below depend on other packages in this workspace that have had changes. Their versions have been incremented to bump the minimum dependency versions of the packages they depend upon in this project. + + - `drift_sqlite_async` - `v0.2.0-alpha.4` + +--- + +#### `sqlite_async` - `v0.11.0` + + - Automatically flush IndexedDB storage to fix durability issues + + ## 2024-11-01 ### Changes diff --git a/packages/drift_sqlite_async/CHANGELOG.md b/packages/drift_sqlite_async/CHANGELOG.md index de46be7..23f501b 100644 --- a/packages/drift_sqlite_async/CHANGELOG.md +++ b/packages/drift_sqlite_async/CHANGELOG.md @@ -1,3 +1,7 @@ +## 0.2.0-alpha.4 + + - Update a dependency to the latest release. + ## 0.2.0-alpha.3 - Bump `sqlite_async` to v0.10.1 diff --git a/packages/drift_sqlite_async/pubspec.yaml b/packages/drift_sqlite_async/pubspec.yaml index 109314d..a2e8c99 100644 --- a/packages/drift_sqlite_async/pubspec.yaml +++ b/packages/drift_sqlite_async/pubspec.yaml @@ -1,5 +1,5 @@ name: drift_sqlite_async -version: 0.2.0-alpha.3 +version: 0.2.0-alpha.4 homepage: https://github.com/powersync-ja/sqlite_async.dart repository: https://github.com/powersync-ja/sqlite_async.dart description: Use Drift with a sqlite_async database, allowing both to be used in the same application. @@ -15,7 +15,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: drift: ">=2.19.0 <3.0.0" - sqlite_async: ^0.10.1 + sqlite_async: ^0.11.0 dev_dependencies: build_runner: ^2.4.8 drift_dev: ">=2.19.0 <3.0.0" diff --git a/packages/sqlite_async/CHANGELOG.md b/packages/sqlite_async/CHANGELOG.md index 88f9875..ab91443 100644 --- a/packages/sqlite_async/CHANGELOG.md +++ b/packages/sqlite_async/CHANGELOG.md @@ -1,3 +1,7 @@ +## 0.11.0 + + - Automatically flush IndexedDB storage to fix durability issues + ## 0.10.1 - For database setups not using a shared worker, use a `BroadcastChannel` to share updates across different tabs. diff --git a/packages/sqlite_async/pubspec.yaml b/packages/sqlite_async/pubspec.yaml index 453f154..09adc13 100644 --- a/packages/sqlite_async/pubspec.yaml +++ b/packages/sqlite_async/pubspec.yaml @@ -1,6 +1,6 @@ name: sqlite_async description: High-performance asynchronous interface for SQLite on Dart and Flutter. -version: 0.10.1 +version: 0.11.0 repository: https://github.com/powersync-ja/sqlite_async.dart environment: sdk: ">=3.4.0 <4.0.0" From 14c1d6f3f0e5ecab3bd0aa630f2eb099026dffda Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Wed, 6 Nov 2024 08:58:14 +0200 Subject: [PATCH 26/90] Fix migrations not running for drift_sqlite_async. --- .../drift_sqlite_async/example/main.g.dart | 131 ++++++++++++++++++ .../example/with_migrations.dart | 15 +- .../example/with_migrations.g.dart | 131 ++++++++++++++++++ .../drift_sqlite_async/lib/src/executor.dart | 6 +- .../test/generated/database.dart | 13 ++ .../test/generated/database.g.dart | 131 ++++++++++++++++++ .../test/migration_test.dart | 40 ++++++ 7 files changed, 457 insertions(+), 10 deletions(-) create mode 100644 packages/drift_sqlite_async/test/migration_test.dart diff --git a/packages/drift_sqlite_async/example/main.g.dart b/packages/drift_sqlite_async/example/main.g.dart index 576157c..0f4385e 100644 --- a/packages/drift_sqlite_async/example/main.g.dart +++ b/packages/drift_sqlite_async/example/main.g.dart @@ -109,6 +109,14 @@ class TodoItem extends DataClass implements Insertable { id: id ?? this.id, description: description ?? this.description, ); + TodoItem copyWithCompanion(TodoItemsCompanion data) { + return TodoItem( + id: data.id.present ? data.id.value : this.id, + description: + data.description.present ? data.description.value : this.description, + ); + } + @override String toString() { return (StringBuffer('TodoItem(') @@ -180,6 +188,7 @@ class TodoItemsCompanion extends UpdateCompanion { abstract class _$AppDatabase extends GeneratedDatabase { _$AppDatabase(QueryExecutor e) : super(e); + $AppDatabaseManager get managers => $AppDatabaseManager(this); late final $TodoItemsTable todoItems = $TodoItemsTable(this); @override Iterable> get allTables => @@ -187,3 +196,125 @@ abstract class _$AppDatabase extends GeneratedDatabase { @override List get allSchemaEntities => [todoItems]; } + +typedef $$TodoItemsTableCreateCompanionBuilder = TodoItemsCompanion Function({ + Value id, + required String description, +}); +typedef $$TodoItemsTableUpdateCompanionBuilder = TodoItemsCompanion Function({ + Value id, + Value description, +}); + +class $$TodoItemsTableFilterComposer + extends Composer<_$AppDatabase, $TodoItemsTable> { + $$TodoItemsTableFilterComposer({ + required super.$db, + required super.$table, + super.joinBuilder, + super.$addJoinBuilderToRootComposer, + super.$removeJoinBuilderFromRootComposer, + }); + ColumnFilters get id => $composableBuilder( + column: $table.id, builder: (column) => ColumnFilters(column)); + + ColumnFilters get description => $composableBuilder( + column: $table.description, builder: (column) => ColumnFilters(column)); +} + +class $$TodoItemsTableOrderingComposer + extends Composer<_$AppDatabase, $TodoItemsTable> { + $$TodoItemsTableOrderingComposer({ + required super.$db, + required super.$table, + super.joinBuilder, + super.$addJoinBuilderToRootComposer, + super.$removeJoinBuilderFromRootComposer, + }); + ColumnOrderings get id => $composableBuilder( + column: $table.id, builder: (column) => ColumnOrderings(column)); + + ColumnOrderings get description => $composableBuilder( + column: $table.description, builder: (column) => ColumnOrderings(column)); +} + +class $$TodoItemsTableAnnotationComposer + extends Composer<_$AppDatabase, $TodoItemsTable> { + $$TodoItemsTableAnnotationComposer({ + required super.$db, + required super.$table, + super.joinBuilder, + super.$addJoinBuilderToRootComposer, + super.$removeJoinBuilderFromRootComposer, + }); + GeneratedColumn get id => + $composableBuilder(column: $table.id, builder: (column) => column); + + GeneratedColumn get description => $composableBuilder( + column: $table.description, builder: (column) => column); +} + +class $$TodoItemsTableTableManager extends RootTableManager< + _$AppDatabase, + $TodoItemsTable, + TodoItem, + $$TodoItemsTableFilterComposer, + $$TodoItemsTableOrderingComposer, + $$TodoItemsTableAnnotationComposer, + $$TodoItemsTableCreateCompanionBuilder, + $$TodoItemsTableUpdateCompanionBuilder, + (TodoItem, BaseReferences<_$AppDatabase, $TodoItemsTable, TodoItem>), + TodoItem, + PrefetchHooks Function()> { + $$TodoItemsTableTableManager(_$AppDatabase db, $TodoItemsTable table) + : super(TableManagerState( + db: db, + table: table, + createFilteringComposer: () => + $$TodoItemsTableFilterComposer($db: db, $table: table), + createOrderingComposer: () => + $$TodoItemsTableOrderingComposer($db: db, $table: table), + createComputedFieldComposer: () => + $$TodoItemsTableAnnotationComposer($db: db, $table: table), + updateCompanionCallback: ({ + Value id = const Value.absent(), + Value description = const Value.absent(), + }) => + TodoItemsCompanion( + id: id, + description: description, + ), + createCompanionCallback: ({ + Value id = const Value.absent(), + required String description, + }) => + TodoItemsCompanion.insert( + id: id, + description: description, + ), + withReferenceMapper: (p0) => p0 + .map((e) => (e.readTable(table), BaseReferences(db, table, e))) + .toList(), + prefetchHooksCallback: null, + )); +} + +typedef $$TodoItemsTableProcessedTableManager = ProcessedTableManager< + _$AppDatabase, + $TodoItemsTable, + TodoItem, + $$TodoItemsTableFilterComposer, + $$TodoItemsTableOrderingComposer, + $$TodoItemsTableAnnotationComposer, + $$TodoItemsTableCreateCompanionBuilder, + $$TodoItemsTableUpdateCompanionBuilder, + (TodoItem, BaseReferences<_$AppDatabase, $TodoItemsTable, TodoItem>), + TodoItem, + PrefetchHooks Function()>; + +class $AppDatabaseManager { + final _$AppDatabase _db; + $AppDatabaseManager(this._db); + $$TodoItemsTableTableManager get todoItems => + $$TodoItemsTableTableManager(_db, _db.todoItems); +} diff --git a/packages/drift_sqlite_async/example/with_migrations.dart b/packages/drift_sqlite_async/example/with_migrations.dart index 2bd4a87..7f8cb44 100644 --- a/packages/drift_sqlite_async/example/with_migrations.dart +++ b/packages/drift_sqlite_async/example/with_migrations.dart @@ -21,21 +21,18 @@ class AppDatabase extends _$AppDatabase { @override MigrationStrategy get migration { - return MigrationStrategy( - onCreate: (m) async { - // In this example, the schema is managed by Drift - await m.createAll(); - }, - ); + return MigrationStrategy(onCreate: (m) async { + // In this example, the schema is managed by Drift. + // For more options, see: + // https://drift.simonbinder.eu/migrations/#usage + await m.createAll(); + }); } } Future main() async { final db = SqliteDatabase(path: 'with_migrations.db'); - await db.execute( - 'CREATE TABLE IF NOT EXISTS todos(id integer primary key, description text)'); - final appdb = AppDatabase(db); // Watch a query on the Drift database diff --git a/packages/drift_sqlite_async/example/with_migrations.g.dart b/packages/drift_sqlite_async/example/with_migrations.g.dart index 67ce020..7c2afbd 100644 --- a/packages/drift_sqlite_async/example/with_migrations.g.dart +++ b/packages/drift_sqlite_async/example/with_migrations.g.dart @@ -109,6 +109,14 @@ class TodoItem extends DataClass implements Insertable { id: id ?? this.id, description: description ?? this.description, ); + TodoItem copyWithCompanion(TodoItemsCompanion data) { + return TodoItem( + id: data.id.present ? data.id.value : this.id, + description: + data.description.present ? data.description.value : this.description, + ); + } + @override String toString() { return (StringBuffer('TodoItem(') @@ -180,6 +188,7 @@ class TodoItemsCompanion extends UpdateCompanion { abstract class _$AppDatabase extends GeneratedDatabase { _$AppDatabase(QueryExecutor e) : super(e); + $AppDatabaseManager get managers => $AppDatabaseManager(this); late final $TodoItemsTable todoItems = $TodoItemsTable(this); @override Iterable> get allTables => @@ -187,3 +196,125 @@ abstract class _$AppDatabase extends GeneratedDatabase { @override List get allSchemaEntities => [todoItems]; } + +typedef $$TodoItemsTableCreateCompanionBuilder = TodoItemsCompanion Function({ + Value id, + required String description, +}); +typedef $$TodoItemsTableUpdateCompanionBuilder = TodoItemsCompanion Function({ + Value id, + Value description, +}); + +class $$TodoItemsTableFilterComposer + extends Composer<_$AppDatabase, $TodoItemsTable> { + $$TodoItemsTableFilterComposer({ + required super.$db, + required super.$table, + super.joinBuilder, + super.$addJoinBuilderToRootComposer, + super.$removeJoinBuilderFromRootComposer, + }); + ColumnFilters get id => $composableBuilder( + column: $table.id, builder: (column) => ColumnFilters(column)); + + ColumnFilters get description => $composableBuilder( + column: $table.description, builder: (column) => ColumnFilters(column)); +} + +class $$TodoItemsTableOrderingComposer + extends Composer<_$AppDatabase, $TodoItemsTable> { + $$TodoItemsTableOrderingComposer({ + required super.$db, + required super.$table, + super.joinBuilder, + super.$addJoinBuilderToRootComposer, + super.$removeJoinBuilderFromRootComposer, + }); + ColumnOrderings get id => $composableBuilder( + column: $table.id, builder: (column) => ColumnOrderings(column)); + + ColumnOrderings get description => $composableBuilder( + column: $table.description, builder: (column) => ColumnOrderings(column)); +} + +class $$TodoItemsTableAnnotationComposer + extends Composer<_$AppDatabase, $TodoItemsTable> { + $$TodoItemsTableAnnotationComposer({ + required super.$db, + required super.$table, + super.joinBuilder, + super.$addJoinBuilderToRootComposer, + super.$removeJoinBuilderFromRootComposer, + }); + GeneratedColumn get id => + $composableBuilder(column: $table.id, builder: (column) => column); + + GeneratedColumn get description => $composableBuilder( + column: $table.description, builder: (column) => column); +} + +class $$TodoItemsTableTableManager extends RootTableManager< + _$AppDatabase, + $TodoItemsTable, + TodoItem, + $$TodoItemsTableFilterComposer, + $$TodoItemsTableOrderingComposer, + $$TodoItemsTableAnnotationComposer, + $$TodoItemsTableCreateCompanionBuilder, + $$TodoItemsTableUpdateCompanionBuilder, + (TodoItem, BaseReferences<_$AppDatabase, $TodoItemsTable, TodoItem>), + TodoItem, + PrefetchHooks Function()> { + $$TodoItemsTableTableManager(_$AppDatabase db, $TodoItemsTable table) + : super(TableManagerState( + db: db, + table: table, + createFilteringComposer: () => + $$TodoItemsTableFilterComposer($db: db, $table: table), + createOrderingComposer: () => + $$TodoItemsTableOrderingComposer($db: db, $table: table), + createComputedFieldComposer: () => + $$TodoItemsTableAnnotationComposer($db: db, $table: table), + updateCompanionCallback: ({ + Value id = const Value.absent(), + Value description = const Value.absent(), + }) => + TodoItemsCompanion( + id: id, + description: description, + ), + createCompanionCallback: ({ + Value id = const Value.absent(), + required String description, + }) => + TodoItemsCompanion.insert( + id: id, + description: description, + ), + withReferenceMapper: (p0) => p0 + .map((e) => (e.readTable(table), BaseReferences(db, table, e))) + .toList(), + prefetchHooksCallback: null, + )); +} + +typedef $$TodoItemsTableProcessedTableManager = ProcessedTableManager< + _$AppDatabase, + $TodoItemsTable, + TodoItem, + $$TodoItemsTableFilterComposer, + $$TodoItemsTableOrderingComposer, + $$TodoItemsTableAnnotationComposer, + $$TodoItemsTableCreateCompanionBuilder, + $$TodoItemsTableUpdateCompanionBuilder, + (TodoItem, BaseReferences<_$AppDatabase, $TodoItemsTable, TodoItem>), + TodoItem, + PrefetchHooks Function()>; + +class $AppDatabaseManager { + final _$AppDatabase _db; + $AppDatabaseManager(this._db); + $$TodoItemsTableTableManager get todoItems => + $$TodoItemsTableTableManager(_db, _db.todoItems); +} diff --git a/packages/drift_sqlite_async/lib/src/executor.dart b/packages/drift_sqlite_async/lib/src/executor.dart index 5d670c1..0727dd8 100644 --- a/packages/drift_sqlite_async/lib/src/executor.dart +++ b/packages/drift_sqlite_async/lib/src/executor.dart @@ -15,6 +15,7 @@ class _SqliteAsyncDelegate extends _SqliteAsyncQueryDelegate implements DatabaseDelegate { final SqliteConnection db; bool _closed = false; + bool _calledOpen = false; _SqliteAsyncDelegate(this.db) : super(db, db.writeLock); @@ -30,12 +31,15 @@ class _SqliteAsyncDelegate extends _SqliteAsyncQueryDelegate _SqliteAsyncTransactionDelegate(db); @override - bool get isOpen => !db.closed && !_closed; + bool get isOpen => !db.closed && !_closed && _calledOpen; @override Future open(QueryExecutorUser user) async { // Workaround - this ensures the db is open await db.get('SELECT 1'); + // We need to delay this until open() has been called, otherwise + // migrations don't run. + _calledOpen = true; } @override diff --git a/packages/drift_sqlite_async/test/generated/database.dart b/packages/drift_sqlite_async/test/generated/database.dart index e955c3d..7747be4 100644 --- a/packages/drift_sqlite_async/test/generated/database.dart +++ b/packages/drift_sqlite_async/test/generated/database.dart @@ -19,3 +19,16 @@ class TodoDatabase extends _$TodoDatabase { @override int get schemaVersion => 1; } + +class TodosMigrationDatabase extends TodoDatabase { + TodosMigrationDatabase(SqliteConnection db) : super(db); + + @override + MigrationStrategy get migration { + return MigrationStrategy( + onCreate: (m) async { + await m.createAll(); + }, + ); + } +} diff --git a/packages/drift_sqlite_async/test/generated/database.g.dart b/packages/drift_sqlite_async/test/generated/database.g.dart index 2572c32..fecba2b 100644 --- a/packages/drift_sqlite_async/test/generated/database.g.dart +++ b/packages/drift_sqlite_async/test/generated/database.g.dart @@ -109,6 +109,14 @@ class TodoItem extends DataClass implements Insertable { id: id ?? this.id, description: description ?? this.description, ); + TodoItem copyWithCompanion(TodoItemsCompanion data) { + return TodoItem( + id: data.id.present ? data.id.value : this.id, + description: + data.description.present ? data.description.value : this.description, + ); + } + @override String toString() { return (StringBuffer('TodoItem(') @@ -180,6 +188,7 @@ class TodoItemsCompanion extends UpdateCompanion { abstract class _$TodoDatabase extends GeneratedDatabase { _$TodoDatabase(QueryExecutor e) : super(e); + $TodoDatabaseManager get managers => $TodoDatabaseManager(this); late final $TodoItemsTable todoItems = $TodoItemsTable(this); @override Iterable> get allTables => @@ -187,3 +196,125 @@ abstract class _$TodoDatabase extends GeneratedDatabase { @override List get allSchemaEntities => [todoItems]; } + +typedef $$TodoItemsTableCreateCompanionBuilder = TodoItemsCompanion Function({ + Value id, + required String description, +}); +typedef $$TodoItemsTableUpdateCompanionBuilder = TodoItemsCompanion Function({ + Value id, + Value description, +}); + +class $$TodoItemsTableFilterComposer + extends Composer<_$TodoDatabase, $TodoItemsTable> { + $$TodoItemsTableFilterComposer({ + required super.$db, + required super.$table, + super.joinBuilder, + super.$addJoinBuilderToRootComposer, + super.$removeJoinBuilderFromRootComposer, + }); + ColumnFilters get id => $composableBuilder( + column: $table.id, builder: (column) => ColumnFilters(column)); + + ColumnFilters get description => $composableBuilder( + column: $table.description, builder: (column) => ColumnFilters(column)); +} + +class $$TodoItemsTableOrderingComposer + extends Composer<_$TodoDatabase, $TodoItemsTable> { + $$TodoItemsTableOrderingComposer({ + required super.$db, + required super.$table, + super.joinBuilder, + super.$addJoinBuilderToRootComposer, + super.$removeJoinBuilderFromRootComposer, + }); + ColumnOrderings get id => $composableBuilder( + column: $table.id, builder: (column) => ColumnOrderings(column)); + + ColumnOrderings get description => $composableBuilder( + column: $table.description, builder: (column) => ColumnOrderings(column)); +} + +class $$TodoItemsTableAnnotationComposer + extends Composer<_$TodoDatabase, $TodoItemsTable> { + $$TodoItemsTableAnnotationComposer({ + required super.$db, + required super.$table, + super.joinBuilder, + super.$addJoinBuilderToRootComposer, + super.$removeJoinBuilderFromRootComposer, + }); + GeneratedColumn get id => + $composableBuilder(column: $table.id, builder: (column) => column); + + GeneratedColumn get description => $composableBuilder( + column: $table.description, builder: (column) => column); +} + +class $$TodoItemsTableTableManager extends RootTableManager< + _$TodoDatabase, + $TodoItemsTable, + TodoItem, + $$TodoItemsTableFilterComposer, + $$TodoItemsTableOrderingComposer, + $$TodoItemsTableAnnotationComposer, + $$TodoItemsTableCreateCompanionBuilder, + $$TodoItemsTableUpdateCompanionBuilder, + (TodoItem, BaseReferences<_$TodoDatabase, $TodoItemsTable, TodoItem>), + TodoItem, + PrefetchHooks Function()> { + $$TodoItemsTableTableManager(_$TodoDatabase db, $TodoItemsTable table) + : super(TableManagerState( + db: db, + table: table, + createFilteringComposer: () => + $$TodoItemsTableFilterComposer($db: db, $table: table), + createOrderingComposer: () => + $$TodoItemsTableOrderingComposer($db: db, $table: table), + createComputedFieldComposer: () => + $$TodoItemsTableAnnotationComposer($db: db, $table: table), + updateCompanionCallback: ({ + Value id = const Value.absent(), + Value description = const Value.absent(), + }) => + TodoItemsCompanion( + id: id, + description: description, + ), + createCompanionCallback: ({ + Value id = const Value.absent(), + required String description, + }) => + TodoItemsCompanion.insert( + id: id, + description: description, + ), + withReferenceMapper: (p0) => p0 + .map((e) => (e.readTable(table), BaseReferences(db, table, e))) + .toList(), + prefetchHooksCallback: null, + )); +} + +typedef $$TodoItemsTableProcessedTableManager = ProcessedTableManager< + _$TodoDatabase, + $TodoItemsTable, + TodoItem, + $$TodoItemsTableFilterComposer, + $$TodoItemsTableOrderingComposer, + $$TodoItemsTableAnnotationComposer, + $$TodoItemsTableCreateCompanionBuilder, + $$TodoItemsTableUpdateCompanionBuilder, + (TodoItem, BaseReferences<_$TodoDatabase, $TodoItemsTable, TodoItem>), + TodoItem, + PrefetchHooks Function()>; + +class $TodoDatabaseManager { + final _$TodoDatabase _db; + $TodoDatabaseManager(this._db); + $$TodoItemsTableTableManager get todoItems => + $$TodoItemsTableTableManager(_db, _db.todoItems); +} diff --git a/packages/drift_sqlite_async/test/migration_test.dart b/packages/drift_sqlite_async/test/migration_test.dart new file mode 100644 index 0000000..e05c212 --- /dev/null +++ b/packages/drift_sqlite_async/test/migration_test.dart @@ -0,0 +1,40 @@ +@TestOn('!browser') +import 'package:sqlite_async/sqlite_async.dart'; +import 'package:test/test.dart'; + +import './utils/test_utils.dart'; +import 'generated/database.dart'; + +void main() { + group('Migration tests', () { + late String path; + late SqliteDatabase db; + late TodoDatabase dbu; + + setUp(() async { + path = dbPath(); + await cleanDb(path: path); + + db = await setupDatabase(path: path); + dbu = TodosMigrationDatabase(db); + }); + + tearDown(() async { + await dbu.close(); + await db.close(); + + await cleanDb(path: path); + }); + + test('INSERT/SELECT', () async { + // This will fail if the migration didn't run + var insertRowId = await dbu + .into(dbu.todoItems) + .insert(TodoItemsCompanion.insert(description: 'Test 1')); + expect(insertRowId, greaterThanOrEqualTo(1)); + + final result = await dbu.select(dbu.todoItems).getSingle(); + expect(result.description, equals('Test 1')); + }); + }); +} From 195182085c7290c2b6760f3e749ffaf9336da07c Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Wed, 6 Nov 2024 09:09:10 +0200 Subject: [PATCH 27/90] Fix tests. --- packages/drift_sqlite_async/README.md | 1 + packages/drift_sqlite_async/test/db_test.dart | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/drift_sqlite_async/README.md b/packages/drift_sqlite_async/README.md index 977dbe1..e8810b1 100644 --- a/packages/drift_sqlite_async/README.md +++ b/packages/drift_sqlite_async/README.md @@ -8,6 +8,7 @@ Supported functionality: 2. Transactions and nested transactions. 3. Table updates are propagated between sqlite_async and Drift - watching queries works using either API. 4. Select queries can run concurrently with writes and other select statements. +5. Drift migrations are supported (optional). ## Usage diff --git a/packages/drift_sqlite_async/test/db_test.dart b/packages/drift_sqlite_async/test/db_test.dart index 6de575e..372f0fa 100644 --- a/packages/drift_sqlite_async/test/db_test.dart +++ b/packages/drift_sqlite_async/test/db_test.dart @@ -54,7 +54,7 @@ void main() { // Drift may or may not emit duplicate update notifications. // We use distinct() to ignore those. .distinct() - .skipWhile((e) => e.isEmpty) + .skipWhile((e) => e == '[]') .take(3) .toList(); @@ -83,7 +83,7 @@ void main() { var resultsPromise = stream .map((rows) => rows.toString()) .distinct() - .skipWhile((e) => e.isEmpty) + .skipWhile((e) => e == '[]') .take(3) .toList(); From efe6fa34e75c33ad8244dedb67c543da244ad6ff Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Wed, 6 Nov 2024 09:30:18 +0200 Subject: [PATCH 28/90] chore(release): publish packages - drift_sqlite_async@0.2.0 --- CHANGELOG.md | 21 +++++++++++++++++++++ packages/drift_sqlite_async/CHANGELOG.md | 4 ++++ packages/drift_sqlite_async/pubspec.yaml | 2 +- 3 files changed, 26 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f5c9c0d..aba9311 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,27 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## 2024-11-06 + +### Changes + +--- + +Packages with breaking changes: + + - There are no breaking changes in this release. + +Packages with other changes: + + - [`drift_sqlite_async` - `v0.2.0`](#drift_sqlite_async---v020) + +--- + +#### `drift_sqlite_async` - `v0.2.0` + + - Automatically run Drift migrations + + ## 2024-11-06 ### Changes diff --git a/packages/drift_sqlite_async/CHANGELOG.md b/packages/drift_sqlite_async/CHANGELOG.md index 23f501b..4c23e85 100644 --- a/packages/drift_sqlite_async/CHANGELOG.md +++ b/packages/drift_sqlite_async/CHANGELOG.md @@ -1,3 +1,7 @@ +## 0.2.0 + + - Automatically run Drift migrations + ## 0.2.0-alpha.4 - Update a dependency to the latest release. diff --git a/packages/drift_sqlite_async/pubspec.yaml b/packages/drift_sqlite_async/pubspec.yaml index a2e8c99..f88dccb 100644 --- a/packages/drift_sqlite_async/pubspec.yaml +++ b/packages/drift_sqlite_async/pubspec.yaml @@ -1,5 +1,5 @@ name: drift_sqlite_async -version: 0.2.0-alpha.4 +version: 0.2.0 homepage: https://github.com/powersync-ja/sqlite_async.dart repository: https://github.com/powersync-ja/sqlite_async.dart description: Use Drift with a sqlite_async database, allowing both to be used in the same application. From 7b4e2a7a7278c788e4148774f1a690cf0abdcfb7 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Thu, 16 Jan 2025 13:47:10 +0100 Subject: [PATCH 29/90] Support running without web workers --- packages/sqlite_async/CHANGELOG.md | 5 ++ .../sqlite_async/lib/src/web/web_mutex.dart | 3 +- .../lib/src/web/web_sqlite_open_factory.dart | 46 ++++++++++++------- .../lib/src/web/worker/worker_utils.dart | 3 +- packages/sqlite_async/lib/web.dart | 18 ++++++++ packages/sqlite_async/pubspec.yaml | 6 +-- 6 files changed, 57 insertions(+), 24 deletions(-) diff --git a/packages/sqlite_async/CHANGELOG.md b/packages/sqlite_async/CHANGELOG.md index ab91443..b49c6ab 100644 --- a/packages/sqlite_async/CHANGELOG.md +++ b/packages/sqlite_async/CHANGELOG.md @@ -1,3 +1,8 @@ +## 0.11.1 + +- Remove remaining `dart:js_util` imports in favor of new interop APIs. +- Add `WebSqliteOpenFactory` with web-specific behavior for open factories. + ## 0.11.0 - Automatically flush IndexedDB storage to fix durability issues diff --git a/packages/sqlite_async/lib/src/web/web_mutex.dart b/packages/sqlite_async/lib/src/web/web_mutex.dart index 38f7ff5..8c2baa5 100644 --- a/packages/sqlite_async/lib/src/web/web_mutex.dart +++ b/packages/sqlite_async/lib/src/web/web_mutex.dart @@ -4,7 +4,6 @@ import 'dart:math'; import 'package:meta/meta.dart'; import 'package:mutex/mutex.dart' as mutex; import 'dart:js_interop'; -import 'dart:js_util' as js_util; // This allows for checking things like hasProperty without the need for depending on the `js` package import 'dart:js_interop_unsafe'; import 'package:web/web.dart'; @@ -128,7 +127,7 @@ class MutexImpl implements Mutex { .request(resolvedIdentifier, lockOptions, jsCallback.toJS); // A timeout abort will throw an exception which needs to be handled. // There should not be any other unhandled lock errors. - js_util.promiseToFuture(promise).catchError((error) {}); + promise.toDart.catchError((error) => null); return gotLock.future; } diff --git a/packages/sqlite_async/lib/src/web/web_sqlite_open_factory.dart b/packages/sqlite_async/lib/src/web/web_sqlite_open_factory.dart index 66000f9..247999d 100644 --- a/packages/sqlite_async/lib/src/web/web_sqlite_open_factory.dart +++ b/packages/sqlite_async/lib/src/web/web_sqlite_open_factory.dart @@ -5,33 +5,45 @@ import 'package:sqlite3_web/sqlite3_web.dart'; import 'package:sqlite_async/sqlite_async.dart'; import 'package:sqlite_async/src/web/database/broadcast_updates.dart'; import 'package:sqlite_async/src/web/web_mutex.dart'; +import 'package:sqlite_async/web.dart'; import 'database.dart'; +import 'worker/worker_utils.dart'; Map> webSQLiteImplementations = {}; /// Web implementation of [AbstractDefaultSqliteOpenFactory] class DefaultSqliteOpenFactory - extends AbstractDefaultSqliteOpenFactory { - final Future _initialized; + extends AbstractDefaultSqliteOpenFactory + implements WebSqliteOpenFactory { + late final Future _initialized = Future.sync(() { + final cacheKey = sqliteOptions.webSqliteOptions.wasmUri + + sqliteOptions.webSqliteOptions.workerUri; + + if (webSQLiteImplementations.containsKey(cacheKey)) { + return webSQLiteImplementations[cacheKey]!; + } + + webSQLiteImplementations[cacheKey] = + openWebSqlite(sqliteOptions.webSqliteOptions); + return webSQLiteImplementations[cacheKey]!; + }); DefaultSqliteOpenFactory( {required super.path, - super.sqliteOptions = const SqliteOptions.defaults()}) - : _initialized = Future.sync(() { - final cacheKey = sqliteOptions.webSqliteOptions.wasmUri + - sqliteOptions.webSqliteOptions.workerUri; - - if (webSQLiteImplementations.containsKey(cacheKey)) { - return webSQLiteImplementations[cacheKey]!; - } - - webSQLiteImplementations[cacheKey] = WebSqlite.open( - wasmModule: Uri.parse(sqliteOptions.webSqliteOptions.wasmUri), - worker: Uri.parse(sqliteOptions.webSqliteOptions.workerUri), - ); - return webSQLiteImplementations[cacheKey]!; - }); + super.sqliteOptions = const SqliteOptions.defaults()}) { + // Make sure initializer starts running immediately + _initialized; + } + + @override + Future openWebSqlite(WebSqliteOptions options) async { + return WebSqlite.open( + wasmModule: Uri.parse(sqliteOptions.webSqliteOptions.wasmUri), + worker: Uri.parse(sqliteOptions.webSqliteOptions.workerUri), + controller: AsyncSqliteController(), + ); + } @override diff --git a/packages/sqlite_async/lib/src/web/worker/worker_utils.dart b/packages/sqlite_async/lib/src/web/worker/worker_utils.dart index 1d8fb5c..5a6e1b3 100644 --- a/packages/sqlite_async/lib/src/web/worker/worker_utils.dart +++ b/packages/sqlite_async/lib/src/web/worker/worker_utils.dart @@ -1,5 +1,4 @@ import 'dart:js_interop'; -import 'dart:js_util' as js_util; import 'package:mutex/mutex.dart'; import 'package:sqlite3/wasm.dart'; @@ -73,7 +72,7 @@ class AsyncSqliteDatabase extends WorkerDatabase { var dartMap = resultSetToMap(res); - var jsObject = js_util.jsify(dartMap); + var jsObject = dartMap.jsify(); return jsObject; case CustomDatabaseMessageKind.executeBatchInTransaction: diff --git a/packages/sqlite_async/lib/web.dart b/packages/sqlite_async/lib/web.dart index a4005e1..c7f9628 100644 --- a/packages/sqlite_async/lib/web.dart +++ b/packages/sqlite_async/lib/web.dart @@ -6,6 +6,8 @@ library sqlite_async.web; import 'package:sqlite3_web/sqlite3_web.dart'; import 'package:web/web.dart'; + +import 'sqlite3_common.dart'; import 'sqlite_async.dart'; import 'src/web/database.dart'; @@ -24,6 +26,22 @@ typedef WebDatabaseEndpoint = ({ String? lockName, }); +/// An additional interface for [SqliteOpenFactory] exposing additional +/// functionality that is only relevant when compiling to the web. +/// +/// The [DefaultSqliteOpenFactory] class implements this interface only when +/// compiling for the web. +abstract interface class WebSqliteOpenFactory + implements SqliteOpenFactory { + /// Opens a [WebSqlite] instance for the given [options]. + /// + /// This method can be overriden in scenarios where the way [WebSqlite] is + /// opened needs to be customized. Implementers should be aware that the + /// result of this method is cached and will be re-used by the open factory + /// when provided with the same [options] again. + Future openWebSqlite(WebSqliteOptions options); +} + /// A [SqliteConnection] interface implemented by opened connections when /// running on the web. /// diff --git a/packages/sqlite_async/pubspec.yaml b/packages/sqlite_async/pubspec.yaml index 09adc13..2c0fff4 100644 --- a/packages/sqlite_async/pubspec.yaml +++ b/packages/sqlite_async/pubspec.yaml @@ -1,6 +1,6 @@ name: sqlite_async description: High-performance asynchronous interface for SQLite on Dart and Flutter. -version: 0.11.0 +version: 0.11.1 repository: https://github.com/powersync-ja/sqlite_async.dart environment: sdk: ">=3.4.0 <4.0.0" @@ -12,8 +12,8 @@ topics: - flutter dependencies: - sqlite3: "^2.4.7" - sqlite3_web: ^0.2.0 + sqlite3: ^2.6.0 + sqlite3_web: ^0.2.1 async: ^2.10.0 collection: ^1.17.0 mutex: ^3.1.0 From 090e6c8d8d7b4e7ae8b43bec0d6dc1551ceb3a8a Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Thu, 16 Jan 2025 14:17:48 +0100 Subject: [PATCH 30/90] Update Dart SDK for tests --- .github/workflows/test.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 2cd34e0..6f6cad4 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -35,19 +35,19 @@ jobs: include: - sqlite_version: "3440200" sqlite_url: "https://www.sqlite.org/2023/sqlite-autoconf-3440200.tar.gz" - dart_sdk: 3.4.0 + dart_sdk: 3.5.0 - sqlite_version: "3430200" sqlite_url: "https://www.sqlite.org/2023/sqlite-autoconf-3430200.tar.gz" - dart_sdk: 3.4.0 + dart_sdk: 3.5.0 - sqlite_version: "3420000" sqlite_url: "https://www.sqlite.org/2023/sqlite-autoconf-3420000.tar.gz" - dart_sdk: 3.4.0 + dart_sdk: 3.5.0 - sqlite_version: "3410100" sqlite_url: "https://www.sqlite.org/2023/sqlite-autoconf-3410100.tar.gz" - dart_sdk: 3.4.0 + dart_sdk: 3.5.0 - sqlite_version: "3380000" sqlite_url: "https://www.sqlite.org/2022/sqlite-autoconf-3380000.tar.gz" - dart_sdk: 3.4.0 + dart_sdk: 3.5.0 steps: - uses: actions/checkout@v3 - uses: dart-lang/setup-dart@v1 From 8baad1ce203a7fd7d4aa8c3ed475435ea42de2c8 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Mon, 20 Jan 2025 09:09:15 +0100 Subject: [PATCH 31/90] Add missing members to throttled database --- .../src/web/worker/throttled_common_database.dart | 13 ++++++++++++- packages/sqlite_async/pubspec.yaml | 2 +- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/packages/sqlite_async/lib/src/web/worker/throttled_common_database.dart b/packages/sqlite_async/lib/src/web/worker/throttled_common_database.dart index ea73bd6..07264bf 100644 --- a/packages/sqlite_async/lib/src/web/worker/throttled_common_database.dart +++ b/packages/sqlite_async/lib/src/web/worker/throttled_common_database.dart @@ -1,6 +1,6 @@ import 'dart:async'; -import 'package:sqlite_async/sqlite3_common.dart'; +import 'package:sqlite_async/sqlite3_wasm.dart'; /// Wrap a CommonDatabase to throttle its updates stream. /// This is so that we can throttle the updates _within_ @@ -103,6 +103,17 @@ class ThrottledCommonDatabase extends CommonDatabase { Stream get updates { return throttledUpdates(_db, _transactionController.stream); } + + @override + VoidPredicate? get commitFilter => _db.commitFilter; + + set commitFilter(VoidPredicate? filter) => _db.commitFilter = filter; + + @override + Stream get commits => _db.commits; + + @override + Stream get rollbacks => _db.rollbacks; } /// This throttles the database update stream to: diff --git a/packages/sqlite_async/pubspec.yaml b/packages/sqlite_async/pubspec.yaml index 2c0fff4..30b296f 100644 --- a/packages/sqlite_async/pubspec.yaml +++ b/packages/sqlite_async/pubspec.yaml @@ -12,7 +12,7 @@ topics: - flutter dependencies: - sqlite3: ^2.6.0 + sqlite3: ^2.7.0 sqlite3_web: ^0.2.1 async: ^2.10.0 collection: ^1.17.0 From 8c770d846cb30e9fdf1ba9cb48ee81019d5f6119 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Tue, 21 Jan 2025 11:22:24 +0100 Subject: [PATCH 32/90] Require upstream version with worker fixes --- packages/sqlite_async/pubspec.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/sqlite_async/pubspec.yaml b/packages/sqlite_async/pubspec.yaml index 30b296f..12c0ad0 100644 --- a/packages/sqlite_async/pubspec.yaml +++ b/packages/sqlite_async/pubspec.yaml @@ -12,8 +12,8 @@ topics: - flutter dependencies: - sqlite3: ^2.7.0 - sqlite3_web: ^0.2.1 + sqlite3: ^2.7.2 + sqlite3_web: ^0.2.2 async: ^2.10.0 collection: ^1.17.0 mutex: ^3.1.0 From e380a06ba791a08c07d16547e213a4ee8f82163f Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Tue, 21 Jan 2025 11:43:28 +0100 Subject: [PATCH 33/90] Remove more deprecated interop APIs --- .../lib/src/impl/isolate_connection_factory_impl.dart | 2 +- packages/sqlite_async/lib/src/impl/mutex_impl.dart | 2 +- .../sqlite_async/lib/src/impl/open_factory_impl.dart | 2 +- .../lib/src/impl/sqlite_database_impl.dart | 2 +- packages/sqlite_async/pubspec.yaml | 1 - packages/sqlite_async/test/utils/test_utils_impl.dart | 2 +- packages/sqlite_async/test/utils/web_test_utils.dart | 10 +++++----- 7 files changed, 10 insertions(+), 11 deletions(-) diff --git a/packages/sqlite_async/lib/src/impl/isolate_connection_factory_impl.dart b/packages/sqlite_async/lib/src/impl/isolate_connection_factory_impl.dart index 4812688..ec70cb0 100644 --- a/packages/sqlite_async/lib/src/impl/isolate_connection_factory_impl.dart +++ b/packages/sqlite_async/lib/src/impl/isolate_connection_factory_impl.dart @@ -2,4 +2,4 @@ export 'stub_isolate_connection_factory.dart' // ignore: uri_does_not_exist if (dart.library.io) '../native/native_isolate_connection_factory.dart' // ignore: uri_does_not_exist - if (dart.library.html) '../web/web_isolate_connection_factory.dart'; + if (dart.library.js_interop) '../web/web_isolate_connection_factory.dart'; diff --git a/packages/sqlite_async/lib/src/impl/mutex_impl.dart b/packages/sqlite_async/lib/src/impl/mutex_impl.dart index 6b77de7..0f45fe4 100644 --- a/packages/sqlite_async/lib/src/impl/mutex_impl.dart +++ b/packages/sqlite_async/lib/src/impl/mutex_impl.dart @@ -2,4 +2,4 @@ export 'stub_mutex.dart' // ignore: uri_does_not_exist if (dart.library.io) '../native/native_isolate_mutex.dart' // ignore: uri_does_not_exist - if (dart.library.html) '../web/web_mutex.dart'; + if (dart.library.js_interop) '../web/web_mutex.dart'; diff --git a/packages/sqlite_async/lib/src/impl/open_factory_impl.dart b/packages/sqlite_async/lib/src/impl/open_factory_impl.dart index e8ce4e1..a7a27ac 100644 --- a/packages/sqlite_async/lib/src/impl/open_factory_impl.dart +++ b/packages/sqlite_async/lib/src/impl/open_factory_impl.dart @@ -2,4 +2,4 @@ export 'stub_sqlite_open_factory.dart' // ignore: uri_does_not_exist if (dart.library.io) '../native/native_sqlite_open_factory.dart' // ignore: uri_does_not_exist - if (dart.library.html) '../web/web_sqlite_open_factory.dart'; + if (dart.library.js_interop) '../web/web_sqlite_open_factory.dart'; diff --git a/packages/sqlite_async/lib/src/impl/sqlite_database_impl.dart b/packages/sqlite_async/lib/src/impl/sqlite_database_impl.dart index a2bcd20..cf1dfbb 100644 --- a/packages/sqlite_async/lib/src/impl/sqlite_database_impl.dart +++ b/packages/sqlite_async/lib/src/impl/sqlite_database_impl.dart @@ -2,4 +2,4 @@ export 'stub_sqlite_database.dart' // ignore: uri_does_not_exist if (dart.library.io) '../native/database/native_sqlite_database.dart' // ignore: uri_does_not_exist - if (dart.library.html) '../web/database/web_sqlite_database.dart'; + if (dart.library.js_interop) '../web/database/web_sqlite_database.dart'; diff --git a/packages/sqlite_async/pubspec.yaml b/packages/sqlite_async/pubspec.yaml index 12c0ad0..0d1127b 100644 --- a/packages/sqlite_async/pubspec.yaml +++ b/packages/sqlite_async/pubspec.yaml @@ -22,7 +22,6 @@ dependencies: dev_dependencies: dcli: ^4.0.0 - js: ^0.6.7 lints: ^3.0.0 test: ^1.21.0 test_api: ^0.7.0 diff --git a/packages/sqlite_async/test/utils/test_utils_impl.dart b/packages/sqlite_async/test/utils/test_utils_impl.dart index 99a34d3..3406d1e 100644 --- a/packages/sqlite_async/test/utils/test_utils_impl.dart +++ b/packages/sqlite_async/test/utils/test_utils_impl.dart @@ -2,4 +2,4 @@ export 'stub_test_utils.dart' // ignore: uri_does_not_exist if (dart.library.io) 'native_test_utils.dart' // ignore: uri_does_not_exist - if (dart.library.html) 'web_test_utils.dart'; + if (dart.library.js_interop) 'web_test_utils.dart'; diff --git a/packages/sqlite_async/test/utils/web_test_utils.dart b/packages/sqlite_async/test/utils/web_test_utils.dart index 2aec683..d25c4e0 100644 --- a/packages/sqlite_async/test/utils/web_test_utils.dart +++ b/packages/sqlite_async/test/utils/web_test_utils.dart @@ -1,9 +1,9 @@ import 'dart:async'; -import 'dart:html'; +import 'dart:js_interop'; -import 'package:js/js.dart'; import 'package:sqlite_async/sqlite_async.dart'; import 'package:test/test.dart'; +import 'package:web/web.dart' show Blob, BlobPart, BlobPropertyBag; import 'abstract_test_utils.dart'; @JS('URL.createObjectURL') @@ -19,13 +19,13 @@ class TestUtils extends AbstractTestUtils { Future _init() async { final channel = spawnHybridUri('/test/server/worker_server.dart'); - final port = await channel.stream.first as int; + final port = (await channel.stream.first as num).toInt(); final sqliteWasmUri = 'http://localhost:$port/sqlite3.wasm'; // Cross origin workers are not supported, but we can supply a Blob var sqliteUri = 'http://localhost:$port/db_worker.js'; - final blob = Blob( - ['importScripts("$sqliteUri");'], 'application/javascript'); + final blob = Blob(['importScripts("$sqliteUri");'.toJS].toJS, + BlobPropertyBag(type: 'application/javascript')); sqliteUri = _createObjectURL(blob); webOptions = SqliteOptions( From f0d684d17b9a0dc0e30c6b91cc5736e91d174620 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Tue, 21 Jan 2025 12:00:31 +0100 Subject: [PATCH 34/90] Revert unintentional change from dart2wasm testing --- packages/sqlite_async/test/utils/web_test_utils.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/sqlite_async/test/utils/web_test_utils.dart b/packages/sqlite_async/test/utils/web_test_utils.dart index d25c4e0..32b7e05 100644 --- a/packages/sqlite_async/test/utils/web_test_utils.dart +++ b/packages/sqlite_async/test/utils/web_test_utils.dart @@ -19,7 +19,7 @@ class TestUtils extends AbstractTestUtils { Future _init() async { final channel = spawnHybridUri('/test/server/worker_server.dart'); - final port = (await channel.stream.first as num).toInt(); + final port = await channel.stream.first as int; final sqliteWasmUri = 'http://localhost:$port/sqlite3.wasm'; // Cross origin workers are not supported, but we can supply a Blob var sqliteUri = 'http://localhost:$port/db_worker.js'; From 31d6b56a2b3617f0d7ff8c5b35324da2fe1b4ca7 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Tue, 21 Jan 2025 15:41:52 +0100 Subject: [PATCH 35/90] Bump minimum SDK version --- packages/sqlite_async/pubspec.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/sqlite_async/pubspec.yaml b/packages/sqlite_async/pubspec.yaml index 0d1127b..1ffa8b4 100644 --- a/packages/sqlite_async/pubspec.yaml +++ b/packages/sqlite_async/pubspec.yaml @@ -3,7 +3,7 @@ description: High-performance asynchronous interface for SQLite on Dart and Flut version: 0.11.1 repository: https://github.com/powersync-ja/sqlite_async.dart environment: - sdk: ">=3.4.0 <4.0.0" + sdk: ">=3.5.0 <4.0.0" topics: - sqlite From 25965185caa10d2a35a747628c7a60dc8e56c024 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Fri, 24 Jan 2025 18:07:17 +0100 Subject: [PATCH 36/90] Fix lints, support dart2wasm --- melos.yaml | 2 +- .../drift_sqlite_async/analysis_options.yaml | 0 .../lib/drift_sqlite_async.dart | 2 - .../drift_sqlite_async/lib/src/executor.dart | 3 +- packages/drift_sqlite_async/pubspec.yaml | 2 + .../drift_sqlite_async/test/basic_test.dart | 2 + packages/drift_sqlite_async/test/db_test.dart | 2 + .../test/generated/database.dart | 2 +- .../test/migration_test.dart | 2 + packages/sqlite_async/analysis_options.yaml | 1 + .../sqlite_async/lib/src/web/database.dart | 14 ++- .../sqlite_async/lib/src/web/protocol.dart | 21 ++++- .../lib/src/web/web_sqlite_open_factory.dart | 7 +- .../web/worker/throttled_common_database.dart | 1 + .../lib/src/web/worker/worker_utils.dart | 51 ++++++++--- packages/sqlite_async/lib/web.dart | 22 ++++- packages/sqlite_async/pubspec.yaml | 10 ++- packages/sqlite_async/test/basic_test.dart | 87 ++++++++++--------- packages/sqlite_async/test/isolate_test.dart | 2 + .../sqlite_async/test/native/basic_test.dart | 2 + .../test/native/native_mutex_test.dart | 2 + .../sqlite_async/test/native/schema_test.dart | 2 + .../sqlite_async/test/native/watch_test.dart | 2 + .../test/server/worker_server.dart | 4 +- .../test/utils/native_test_utils.dart | 5 +- .../test/utils/web_test_utils.dart | 2 +- .../sqlite_async/test/web/watch_test.dart | 2 + scripts/sqlite3_wasm_download.dart | 2 + 28 files changed, 173 insertions(+), 83 deletions(-) rename analysis_options.yaml => packages/drift_sqlite_async/analysis_options.yaml (100%) create mode 100644 packages/sqlite_async/analysis_options.yaml diff --git a/melos.yaml b/melos.yaml index 8de8bfe..875e7a5 100644 --- a/melos.yaml +++ b/melos.yaml @@ -41,7 +41,7 @@ scripts: test: description: Run tests in a specific package. - run: dart test -p chrome,vm + run: dart test -p chrome,vm --compilers dart2js,dart2wasm exec: concurrency: 1 packageFilters: diff --git a/analysis_options.yaml b/packages/drift_sqlite_async/analysis_options.yaml similarity index 100% rename from analysis_options.yaml rename to packages/drift_sqlite_async/analysis_options.yaml diff --git a/packages/drift_sqlite_async/lib/drift_sqlite_async.dart b/packages/drift_sqlite_async/lib/drift_sqlite_async.dart index f842c5b..83d04fc 100644 --- a/packages/drift_sqlite_async/lib/drift_sqlite_async.dart +++ b/packages/drift_sqlite_async/lib/drift_sqlite_async.dart @@ -1,4 +1,2 @@ -library drift_sqlite_async; - export './src/connection.dart'; export './src/executor.dart'; diff --git a/packages/drift_sqlite_async/lib/src/executor.dart b/packages/drift_sqlite_async/lib/src/executor.dart index 0727dd8..dbd4c96 100644 --- a/packages/drift_sqlite_async/lib/src/executor.dart +++ b/packages/drift_sqlite_async/lib/src/executor.dart @@ -1,7 +1,7 @@ import 'dart:async'; import 'package:drift/backends.dart'; -import 'package:drift/src/runtime/query_builder/query_builder.dart'; +import 'package:drift/drift.dart'; import 'package:sqlite_async/sqlite3_common.dart'; import 'package:sqlite_async/sqlite_async.dart'; @@ -19,6 +19,7 @@ class _SqliteAsyncDelegate extends _SqliteAsyncQueryDelegate _SqliteAsyncDelegate(this.db) : super(db, db.writeLock); + @override bool isInTransaction = false; // unused @override diff --git a/packages/drift_sqlite_async/pubspec.yaml b/packages/drift_sqlite_async/pubspec.yaml index f88dccb..197f9b8 100644 --- a/packages/drift_sqlite_async/pubspec.yaml +++ b/packages/drift_sqlite_async/pubspec.yaml @@ -16,10 +16,12 @@ environment: dependencies: drift: ">=2.19.0 <3.0.0" sqlite_async: ^0.11.0 + dev_dependencies: build_runner: ^2.4.8 drift_dev: ">=2.19.0 <3.0.0" glob: ^2.1.2 + lints: ^5.1.1 sqlite3: ^2.4.0 test: ^1.25.2 test_api: ^0.7.0 diff --git a/packages/drift_sqlite_async/test/basic_test.dart b/packages/drift_sqlite_async/test/basic_test.dart index 1b33562..371c149 100644 --- a/packages/drift_sqlite_async/test/basic_test.dart +++ b/packages/drift_sqlite_async/test/basic_test.dart @@ -1,5 +1,7 @@ // TODO @TestOn('!browser') +library; + import 'dart:async'; import 'package:drift/drift.dart'; diff --git a/packages/drift_sqlite_async/test/db_test.dart b/packages/drift_sqlite_async/test/db_test.dart index 372f0fa..39a5224 100644 --- a/packages/drift_sqlite_async/test/db_test.dart +++ b/packages/drift_sqlite_async/test/db_test.dart @@ -1,5 +1,7 @@ // TODO @TestOn('!browser') +library; + import 'package:drift/drift.dart'; import 'package:sqlite_async/sqlite_async.dart'; import 'package:test/test.dart'; diff --git a/packages/drift_sqlite_async/test/generated/database.dart b/packages/drift_sqlite_async/test/generated/database.dart index 7747be4..928c7dd 100644 --- a/packages/drift_sqlite_async/test/generated/database.dart +++ b/packages/drift_sqlite_async/test/generated/database.dart @@ -21,7 +21,7 @@ class TodoDatabase extends _$TodoDatabase { } class TodosMigrationDatabase extends TodoDatabase { - TodosMigrationDatabase(SqliteConnection db) : super(db); + TodosMigrationDatabase(super.db); @override MigrationStrategy get migration { diff --git a/packages/drift_sqlite_async/test/migration_test.dart b/packages/drift_sqlite_async/test/migration_test.dart index e05c212..ca9cb3f 100644 --- a/packages/drift_sqlite_async/test/migration_test.dart +++ b/packages/drift_sqlite_async/test/migration_test.dart @@ -1,4 +1,6 @@ @TestOn('!browser') +library; + import 'package:sqlite_async/sqlite_async.dart'; import 'package:test/test.dart'; diff --git a/packages/sqlite_async/analysis_options.yaml b/packages/sqlite_async/analysis_options.yaml new file mode 100644 index 0000000..572dd23 --- /dev/null +++ b/packages/sqlite_async/analysis_options.yaml @@ -0,0 +1 @@ +include: package:lints/recommended.yaml diff --git a/packages/sqlite_async/lib/src/web/database.dart b/packages/sqlite_async/lib/src/web/database.dart index a4f0ddf..dc3a7af 100644 --- a/packages/sqlite_async/lib/src/web/database.dart +++ b/packages/sqlite_async/lib/src/web/database.dart @@ -1,8 +1,10 @@ import 'dart:async'; import 'dart:js_interop'; +import 'dart:js_interop_unsafe'; import 'package:sqlite3/common.dart'; import 'package:sqlite3_web/sqlite3_web.dart'; +import 'package:sqlite3_web/protocol_utils.dart' as proto; import 'package:sqlite_async/sqlite_async.dart'; import 'package:sqlite_async/src/utils/shared_utils.dart'; import 'package:sqlite_async/src/web/database/broadcast_updates.dart'; @@ -256,9 +258,15 @@ class _ExclusiveTransactionContext extends _ExclusiveContext { // JavaScript object. This is the converted into a Dart ResultSet. return await wrapSqliteException(() async { var res = await _database._database.customRequest(CustomDatabaseMessage( - CustomDatabaseMessageKind.executeInTransaction, sql, parameters)); - var result = - Map.from((res as JSObject).dartify() as Map); + CustomDatabaseMessageKind.executeInTransaction, sql, parameters)) + as JSObject; + + if (res.has('format') && (res['format'] as JSNumber).toDartInt == 2) { + // Newer workers use a serialization format more efficient than dartify(). + return proto.deserializeResultSet(res['r'] as JSObject); + } + + var result = Map.from(res.dartify() as Map); final columnNames = [ for (final entry in result['columnNames']) entry as String ]; diff --git a/packages/sqlite_async/lib/src/web/protocol.dart b/packages/sqlite_async/lib/src/web/protocol.dart index e6206d6..a20d4f5 100644 --- a/packages/sqlite_async/lib/src/web/protocol.dart +++ b/packages/sqlite_async/lib/src/web/protocol.dart @@ -3,6 +3,7 @@ library; import 'dart:js_interop'; +import 'package:sqlite3_web/protocol_utils.dart' as proto; enum CustomDatabaseMessageKind { requestSharedLock, @@ -19,15 +20,24 @@ extension type CustomDatabaseMessage._raw(JSObject _) implements JSObject { required JSString rawKind, JSString rawSql, JSArray rawParameters, + JSArrayBuffer typeInfo, }); factory CustomDatabaseMessage(CustomDatabaseMessageKind kind, [String? sql, List parameters = const []]) { - final rawSql = sql?.toJS ?? ''.toJS; - final rawParameters = - [for (final parameter in parameters) parameter.jsify()].toJS; + final rawSql = (sql ?? '').toJS; + // Serializing parameters this was is backwards-compatible with dartify() + // on the other end, but a bit more efficient while also suppporting sound + // communcation between dart2js workers and dart2wasm clients. + // Older workers ignore the typeInfo, but that's not a problem. + final (rawParameters, typeInfo) = proto.serializeParameters(parameters); + return CustomDatabaseMessage._( - rawKind: kind.name.toJS, rawSql: rawSql, rawParameters: rawParameters); + rawKind: kind.name.toJS, + rawSql: rawSql, + rawParameters: rawParameters, + typeInfo: typeInfo, + ); } external JSString get rawKind; @@ -36,6 +46,9 @@ extension type CustomDatabaseMessage._raw(JSObject _) implements JSObject { external JSArray get rawParameters; + /// Not set in earlier versions of this package. + external JSArrayBuffer? get typeInfo; + CustomDatabaseMessageKind get kind { return CustomDatabaseMessageKind.values.byName(rawKind.toDart); } diff --git a/packages/sqlite_async/lib/src/web/web_sqlite_open_factory.dart b/packages/sqlite_async/lib/src/web/web_sqlite_open_factory.dart index 247999d..5089b95 100644 --- a/packages/sqlite_async/lib/src/web/web_sqlite_open_factory.dart +++ b/packages/sqlite_async/lib/src/web/web_sqlite_open_factory.dart @@ -15,7 +15,7 @@ Map> webSQLiteImplementations = {}; /// Web implementation of [AbstractDefaultSqliteOpenFactory] class DefaultSqliteOpenFactory extends AbstractDefaultSqliteOpenFactory - implements WebSqliteOpenFactory { + with WebSqliteOpenFactory { late final Future _initialized = Future.sync(() { final cacheKey = sqliteOptions.webSqliteOptions.wasmUri + sqliteOptions.webSqliteOptions.workerUri; @@ -45,9 +45,8 @@ class DefaultSqliteOpenFactory ); } - @override - /// This is currently not supported on web + @override CommonDatabase openDB(SqliteOpenOptions options) { throw UnimplementedError( 'Direct access to CommonDatabase is not available on web.'); @@ -61,7 +60,7 @@ class DefaultSqliteOpenFactory /// Due to being asynchronous, the under laying CommonDatabase is not accessible Future openConnection(SqliteOpenOptions options) async { final workers = await _initialized; - final connection = await workers.connectToRecommended(path); + final connection = await connectToWorker(workers, path); // When the database is accessed through a shared worker, we implement // mutexes over custom messages sent through the shared worker. In other diff --git a/packages/sqlite_async/lib/src/web/worker/throttled_common_database.dart b/packages/sqlite_async/lib/src/web/worker/throttled_common_database.dart index 07264bf..5f33b6d 100644 --- a/packages/sqlite_async/lib/src/web/worker/throttled_common_database.dart +++ b/packages/sqlite_async/lib/src/web/worker/throttled_common_database.dart @@ -107,6 +107,7 @@ class ThrottledCommonDatabase extends CommonDatabase { @override VoidPredicate? get commitFilter => _db.commitFilter; + @override set commitFilter(VoidPredicate? filter) => _db.commitFilter = filter; @override diff --git a/packages/sqlite_async/lib/src/web/worker/worker_utils.dart b/packages/sqlite_async/lib/src/web/worker/worker_utils.dart index 5a6e1b3..af39747 100644 --- a/packages/sqlite_async/lib/src/web/worker/worker_utils.dart +++ b/packages/sqlite_async/lib/src/web/worker/worker_utils.dart @@ -1,8 +1,12 @@ import 'dart:js_interop'; +import 'dart:js_interop_unsafe'; +import 'package:meta/meta.dart'; import 'package:mutex/mutex.dart'; import 'package:sqlite3/wasm.dart'; import 'package:sqlite3_web/sqlite3_web.dart'; +import 'package:sqlite3_web/protocol_utils.dart' as proto; + import 'throttled_common_database.dart'; import '../protocol.dart'; @@ -12,9 +16,9 @@ import '../protocol.dart'; /// can be extended to perform custom requests. base class AsyncSqliteController extends DatabaseController { @override - Future openDatabase( - WasmSqlite3 sqlite3, String path, String vfs) async { - final db = sqlite3.open(path, vfs: vfs); + Future openDatabase(WasmSqlite3 sqlite3, String path, + String vfs, JSAny? additionalData) async { + final db = openUnderlying(sqlite3, path, vfs, additionalData); // Register any custom functions here if needed @@ -23,6 +27,18 @@ base class AsyncSqliteController extends DatabaseController { return AsyncSqliteDatabase(database: throttled); } + /// Opens a database with the `sqlite3` package that will be wrapped in a + /// [ThrottledCommonDatabase] for [openDatabase]. + @visibleForOverriding + CommonDatabase openUnderlying( + WasmSqlite3 sqlite3, + String path, + String vfs, + JSAny? additionalData, + ) { + return sqlite3.open(path, vfs: vfs); + } + @override Future handleCustomRequest( ClientConnection connection, JSAny? request) { @@ -61,25 +77,32 @@ class AsyncSqliteDatabase extends WorkerDatabase { return database.autocommit.toJS; case CustomDatabaseMessageKind.executeInTransaction: final sql = message.rawSql.toDart; - final parameters = [ - for (final raw in (message.rawParameters).toDart) raw.dartify() - ]; + final hasTypeInfo = message.typeInfo.isDefinedAndNotNull; + final parameters = proto.deserializeParameters( + message.rawParameters, message.typeInfo); if (database.autocommit) { throw SqliteException(0, "Transaction rolled back by earlier statement. Cannot execute: $sql"); } - var res = database.select(sql, parameters); - var dartMap = resultSetToMap(res); - - var jsObject = dartMap.jsify(); + var res = database.select(sql, parameters); + if (hasTypeInfo) { + // If the client is sending a request that has parameters with type + // information, it will also support a newer serialization format for + // result sets. + return JSObject() + ..['format'] = 2.toJS + ..['r'] = proto.serializeResultSet(res); + } else { + var dartMap = resultSetToMap(res); + var jsObject = dartMap.jsify(); + return jsObject; + } - return jsObject; case CustomDatabaseMessageKind.executeBatchInTransaction: final sql = message.rawSql.toDart; - final parameters = [ - for (final raw in (message.rawParameters).toDart) raw.dartify() - ]; + final parameters = proto.deserializeParameters( + message.rawParameters, message.typeInfo); if (database.autocommit) { throw SqliteException(0, "Transaction rolled back by earlier statement. Cannot execute: $sql"); diff --git a/packages/sqlite_async/lib/web.dart b/packages/sqlite_async/lib/web.dart index c7f9628..8051edb 100644 --- a/packages/sqlite_async/lib/web.dart +++ b/packages/sqlite_async/lib/web.dart @@ -2,7 +2,7 @@ /// /// These expose methods allowing database instances to be shared across web /// workers. -library sqlite_async.web; +library; import 'package:sqlite3_web/sqlite3_web.dart'; import 'package:web/web.dart'; @@ -31,7 +31,7 @@ typedef WebDatabaseEndpoint = ({ /// /// The [DefaultSqliteOpenFactory] class implements this interface only when /// compiling for the web. -abstract interface class WebSqliteOpenFactory +abstract mixin class WebSqliteOpenFactory implements SqliteOpenFactory { /// Opens a [WebSqlite] instance for the given [options]. /// @@ -39,7 +39,21 @@ abstract interface class WebSqliteOpenFactory /// opened needs to be customized. Implementers should be aware that the /// result of this method is cached and will be re-used by the open factory /// when provided with the same [options] again. - Future openWebSqlite(WebSqliteOptions options); + Future openWebSqlite(WebSqliteOptions options) async { + return WebSqlite.open( + worker: Uri.parse(options.workerUri), + wasmModule: Uri.parse(options.wasmUri), + ); + } + + /// Uses [WebSqlite] to connects to the recommended database setup for [name]. + /// + /// This typically just calls [WebSqlite.connectToRecommended], but subclasses + /// can customize the behavior where needed. + Future connectToWorker( + WebSqlite sqlite, String name) { + return sqlite.connectToRecommended(name); + } } /// A [SqliteConnection] interface implemented by opened connections when @@ -91,6 +105,7 @@ abstract class WebSqliteConnection implements SqliteConnection { /// This only has an effect when IndexedDB storage is used. /// /// See [flush] for details. + @override Future writeLock(Future Function(SqliteWriteContext tx) callback, {Duration? lockTimeout, String? debugContext, bool? flush}); @@ -101,6 +116,7 @@ abstract class WebSqliteConnection implements SqliteConnection { /// This only has an effect when IndexedDB storage is used. /// /// See [flush] for details. + @override Future writeTransaction( Future Function(SqliteWriteContext tx) callback, {Duration? lockTimeout, diff --git a/packages/sqlite_async/pubspec.yaml b/packages/sqlite_async/pubspec.yaml index 1ffa8b4..0a616b9 100644 --- a/packages/sqlite_async/pubspec.yaml +++ b/packages/sqlite_async/pubspec.yaml @@ -13,7 +13,7 @@ topics: dependencies: sqlite3: ^2.7.2 - sqlite3_web: ^0.2.2 + sqlite3_web: ^0.3.0 async: ^2.10.0 collection: ^1.17.0 mutex: ^3.1.0 @@ -21,8 +21,7 @@ dependencies: web: ^1.0.0 dev_dependencies: - dcli: ^4.0.0 - lints: ^3.0.0 + lints: ^5.1.1 test: ^1.21.0 test_api: ^0.7.0 glob: ^2.1.1 @@ -31,6 +30,11 @@ dev_dependencies: shelf_static: ^1.1.2 stream_channel: ^2.1.2 path: ^1.9.0 + test_descriptor: ^2.0.2 + +dependency_overrides: + sqlite3_web: + path: /Users/simon/src/sqlite3.dart/sqlite3_web platforms: android: diff --git a/packages/sqlite_async/test/basic_test.dart b/packages/sqlite_async/test/basic_test.dart index e07daf4..83169fe 100644 --- a/packages/sqlite_async/test/basic_test.dart +++ b/packages/sqlite_async/test/basic_test.dart @@ -6,6 +6,7 @@ import 'package:test/test.dart'; import 'utils/test_utils_impl.dart'; final testUtils = TestUtils(); +const _isDart2Wasm = bool.fromEnvironment('dart.tool.dart2wasm'); void main() { group('Shared Basic Tests', () { @@ -125,49 +126,55 @@ void main() { expect(savedTx!.closed, equals(true)); }); - test('should properly report errors in transactions', () async { - final db = await testUtils.setupDatabase(path: path); - await createTables(db); + test( + 'should properly report errors in transactions', + () async { + final db = await testUtils.setupDatabase(path: path); + await createTables(db); - var tp = db.writeTransaction((tx) async { - await tx.execute( - 'INSERT OR ROLLBACK INTO test_data(id, description) VALUES(?, ?)', - [1, 'test1']); - await tx.execute( - 'INSERT OR ROLLBACK INTO test_data(id, description) VALUES(?, ?)', - [2, 'test2']); - expect(await tx.getAutoCommit(), equals(false)); - try { + var tp = db.writeTransaction((tx) async { await tx.execute( 'INSERT OR ROLLBACK INTO test_data(id, description) VALUES(?, ?)', - [2, 'test3']); - } catch (e) { - // Ignore - } - - expect(await tx.getAutoCommit(), equals(true)); - expect(tx.closed, equals(false)); - - // Will not be executed because of the above rollback - await tx.execute( - 'INSERT OR ROLLBACK INTO test_data(id, description) VALUES(?, ?)', - [4, 'test4']); - }); - - // The error propagates up to the transaction - await expectLater( - tp, - throwsA((e) => - e is SqliteException && - e.message - .contains('Transaction rolled back by earlier statement'))); - - expect(await db.get('SELECT count() count FROM test_data'), - equals({'count': 0})); - - // Check that we can open another transaction afterwards - await db.writeTransaction((tx) async {}); - }); + [1, 'test1']); + await tx.execute( + 'INSERT OR ROLLBACK INTO test_data(id, description) VALUES(?, ?)', + [2, 'test2']); + expect(await tx.getAutoCommit(), equals(false)); + try { + await tx.execute( + 'INSERT OR ROLLBACK INTO test_data(id, description) VALUES(?, ?)', + [2, 'test3']); + } catch (e) { + // Ignore + } + + expect(await tx.getAutoCommit(), equals(true)); + expect(tx.closed, equals(false)); + + // Will not be executed because of the above rollback + await tx.execute( + 'INSERT OR ROLLBACK INTO test_data(id, description) VALUES(?, ?)', + [4, 'test4']); + }); + + // The error propagates up to the transaction + await expectLater( + tp, + throwsA((e) => + e is SqliteException && + e.message + .contains('Transaction rolled back by earlier statement'))); + + expect(await db.get('SELECT count() count FROM test_data'), + equals({'count': 0})); + + // Check that we can open another transaction afterwards + await db.writeTransaction((tx) async {}); + }, + skip: _isDart2Wasm + ? 'Fails due to compiler bug, https://dartbug.com/59981' + : null, + ); }); } diff --git a/packages/sqlite_async/test/isolate_test.dart b/packages/sqlite_async/test/isolate_test.dart index 60bea87..3a3af98 100644 --- a/packages/sqlite_async/test/isolate_test.dart +++ b/packages/sqlite_async/test/isolate_test.dart @@ -1,4 +1,6 @@ @TestOn('!browser') +library; + import 'dart:isolate'; import 'package:test/test.dart'; diff --git a/packages/sqlite_async/test/native/basic_test.dart b/packages/sqlite_async/test/native/basic_test.dart index 263ab39..eba5493 100644 --- a/packages/sqlite_async/test/native/basic_test.dart +++ b/packages/sqlite_async/test/native/basic_test.dart @@ -1,4 +1,6 @@ @TestOn('!browser') +library; + import 'dart:async'; import 'dart:math'; diff --git a/packages/sqlite_async/test/native/native_mutex_test.dart b/packages/sqlite_async/test/native/native_mutex_test.dart index 699a877..e9b5a54 100644 --- a/packages/sqlite_async/test/native/native_mutex_test.dart +++ b/packages/sqlite_async/test/native/native_mutex_test.dart @@ -1,4 +1,6 @@ @TestOn('!browser') +library; + import 'dart:isolate'; import 'package:sqlite_async/src/native/native_isolate_mutex.dart'; diff --git a/packages/sqlite_async/test/native/schema_test.dart b/packages/sqlite_async/test/native/schema_test.dart index c358402..423d33b 100644 --- a/packages/sqlite_async/test/native/schema_test.dart +++ b/packages/sqlite_async/test/native/schema_test.dart @@ -1,4 +1,6 @@ @TestOn('!browser') +library; + import 'dart:async'; import 'package:sqlite_async/sqlite_async.dart'; diff --git a/packages/sqlite_async/test/native/watch_test.dart b/packages/sqlite_async/test/native/watch_test.dart index 67077db..4e4fb83 100644 --- a/packages/sqlite_async/test/native/watch_test.dart +++ b/packages/sqlite_async/test/native/watch_test.dart @@ -1,4 +1,6 @@ @TestOn('!browser') +library; + import 'dart:async'; import 'dart:isolate'; import 'dart:math'; diff --git a/packages/sqlite_async/test/server/worker_server.dart b/packages/sqlite_async/test/server/worker_server.dart index 30cffe9..4d060e2 100644 --- a/packages/sqlite_async/test/server/worker_server.dart +++ b/packages/sqlite_async/test/server/worker_server.dart @@ -1,6 +1,5 @@ import 'dart:io'; -import 'package:dcli/dcli.dart'; import 'package:path/path.dart' as p; import 'package:shelf/shelf.dart'; import 'package:shelf/shelf_io.dart' as io; @@ -10,8 +9,7 @@ import 'package:stream_channel/stream_channel.dart'; import 'asset_server.dart'; Future hybridMain(StreamChannel channel) async { - final directory = p.normalize( - p.join(DartScript.self.pathToScriptDirectory, '../../../../assets')); + final directory = p.normalize('../../assets'); final sqliteOutputPath = p.join(directory, 'sqlite3.wasm'); diff --git a/packages/sqlite_async/test/utils/native_test_utils.dart b/packages/sqlite_async/test/utils/native_test_utils.dart index e23bb65..31db65c 100644 --- a/packages/sqlite_async/test/utils/native_test_utils.dart +++ b/packages/sqlite_async/test/utils/native_test_utils.dart @@ -8,6 +8,7 @@ import 'package:glob/list_local_fs.dart'; import 'package:sqlite_async/sqlite3_common.dart'; import 'package:sqlite_async/sqlite_async.dart'; import 'package:sqlite3/open.dart' as sqlite_open; +import 'package:test_descriptor/test_descriptor.dart' as d; import 'abstract_test_utils.dart'; @@ -50,10 +51,8 @@ class TestSqliteOpenFactory extends TestDefaultSqliteOpenFactory { } class TestUtils extends AbstractTestUtils { - @override String dbPath() { - Directory("test-db").createSync(recursive: false); - return super.dbPath(); + return d.path('test.db'); } @override diff --git a/packages/sqlite_async/test/utils/web_test_utils.dart b/packages/sqlite_async/test/utils/web_test_utils.dart index 32b7e05..d25c4e0 100644 --- a/packages/sqlite_async/test/utils/web_test_utils.dart +++ b/packages/sqlite_async/test/utils/web_test_utils.dart @@ -19,7 +19,7 @@ class TestUtils extends AbstractTestUtils { Future _init() async { final channel = spawnHybridUri('/test/server/worker_server.dart'); - final port = await channel.stream.first as int; + final port = (await channel.stream.first as num).toInt(); final sqliteWasmUri = 'http://localhost:$port/sqlite3.wasm'; // Cross origin workers are not supported, but we can supply a Blob var sqliteUri = 'http://localhost:$port/db_worker.js'; diff --git a/packages/sqlite_async/test/web/watch_test.dart b/packages/sqlite_async/test/web/watch_test.dart index 50bbae0..c6757b9 100644 --- a/packages/sqlite_async/test/web/watch_test.dart +++ b/packages/sqlite_async/test/web/watch_test.dart @@ -1,4 +1,6 @@ @TestOn('browser') +library; + import 'package:sqlite_async/sqlite_async.dart'; import 'package:test/test.dart'; diff --git a/scripts/sqlite3_wasm_download.dart b/scripts/sqlite3_wasm_download.dart index 28d91b4..62acbbe 100644 --- a/scripts/sqlite3_wasm_download.dart +++ b/scripts/sqlite3_wasm_download.dart @@ -1,4 +1,6 @@ /// Downloads sqlite3.wasm +library; + import 'dart:io'; final sqliteUrl = From 01d63692a01bbf4dbbb606cc085a8ab72cfcc475 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Fri, 24 Jan 2025 22:42:11 +0100 Subject: [PATCH 37/90] Avoid test implementation import --- .../test/utils/abstract_test_utils.dart | 10 +--------- .../test/utils/native_test_utils.dart | 4 ++++ .../test/utils/stub_test_utils.dart | 5 +++++ .../test/utils/web_test_utils.dart | 18 ++++++++++++++++++ 4 files changed, 28 insertions(+), 9 deletions(-) diff --git a/packages/sqlite_async/test/utils/abstract_test_utils.dart b/packages/sqlite_async/test/utils/abstract_test_utils.dart index f1ec6ea..e787a45 100644 --- a/packages/sqlite_async/test/utils/abstract_test_utils.dart +++ b/packages/sqlite_async/test/utils/abstract_test_utils.dart @@ -1,5 +1,4 @@ import 'package:sqlite_async/sqlite_async.dart'; -import 'package:test_api/src/backend/invoker.dart'; class TestDefaultSqliteOpenFactory extends DefaultSqliteOpenFactory { final String sqlitePath; @@ -9,14 +8,7 @@ class TestDefaultSqliteOpenFactory extends DefaultSqliteOpenFactory { } abstract class AbstractTestUtils { - String dbPath() { - final test = Invoker.current!.liveTest; - var testName = test.test.name; - var testShortName = - testName.replaceAll(RegExp(r'[\s\./]'), '_').toLowerCase(); - var dbName = "test-db/$testShortName.db"; - return dbName; - } + String dbPath(); /// Generates a test open factory Future testFactory( diff --git a/packages/sqlite_async/test/utils/native_test_utils.dart b/packages/sqlite_async/test/utils/native_test_utils.dart index 31db65c..45e0500 100644 --- a/packages/sqlite_async/test/utils/native_test_utils.dart +++ b/packages/sqlite_async/test/utils/native_test_utils.dart @@ -26,6 +26,10 @@ class TestSqliteOpenFactory extends TestDefaultSqliteOpenFactory { sqlite_open.open.overrideFor(sqlite_open.OperatingSystem.linux, () { return DynamicLibrary.open(sqlitePath); }); + sqlite_open.open.overrideFor(sqlite_open.OperatingSystem.macOS, () { + return DynamicLibrary.open( + '/Users/simon/src/sqlite_async.dart/sqlite-autoconf-3480000/.libs/libsqlite3.dylib'); + }); final db = super.open(options); db.createFunction( diff --git a/packages/sqlite_async/test/utils/stub_test_utils.dart b/packages/sqlite_async/test/utils/stub_test_utils.dart index 5e3a953..852009f 100644 --- a/packages/sqlite_async/test/utils/stub_test_utils.dart +++ b/packages/sqlite_async/test/utils/stub_test_utils.dart @@ -1,6 +1,11 @@ import 'abstract_test_utils.dart'; class TestUtils extends AbstractTestUtils { + @override + String dbPath() { + throw UnimplementedError(); + } + @override Future cleanDb({required String path}) { throw UnimplementedError(); diff --git a/packages/sqlite_async/test/utils/web_test_utils.dart b/packages/sqlite_async/test/utils/web_test_utils.dart index d25c4e0..ac718f2 100644 --- a/packages/sqlite_async/test/utils/web_test_utils.dart +++ b/packages/sqlite_async/test/utils/web_test_utils.dart @@ -1,5 +1,6 @@ import 'dart:async'; import 'dart:js_interop'; +import 'dart:math'; import 'package:sqlite_async/sqlite_async.dart'; import 'package:test/test.dart'; @@ -9,6 +10,8 @@ import 'abstract_test_utils.dart'; @JS('URL.createObjectURL') external String _createObjectURL(Blob blob); +String? _dbPath; + class TestUtils extends AbstractTestUtils { late Future _isInitialized; late final SqliteOptions webOptions; @@ -33,6 +36,21 @@ class TestUtils extends AbstractTestUtils { wasmUri: sqliteWasmUri.toString(), workerUri: sqliteUri)); } + @override + String dbPath() { + if (_dbPath case final path?) { + return path; + } + + final created = _dbPath = 'test-db/${Random().nextInt(1 << 31)}/test.db'; + addTearDown(() { + // Pick a new path for the next test. + _dbPath = null; + }); + + return created; + } + @override Future cleanDb({required String path}) async {} From 29d1d8dc19e4369b361aff29569cf6364685b1b7 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Fri, 24 Jan 2025 22:43:42 +0100 Subject: [PATCH 38/90] Remove unintentional macos override --- packages/sqlite_async/test/utils/native_test_utils.dart | 4 ---- 1 file changed, 4 deletions(-) diff --git a/packages/sqlite_async/test/utils/native_test_utils.dart b/packages/sqlite_async/test/utils/native_test_utils.dart index 45e0500..31db65c 100644 --- a/packages/sqlite_async/test/utils/native_test_utils.dart +++ b/packages/sqlite_async/test/utils/native_test_utils.dart @@ -26,10 +26,6 @@ class TestSqliteOpenFactory extends TestDefaultSqliteOpenFactory { sqlite_open.open.overrideFor(sqlite_open.OperatingSystem.linux, () { return DynamicLibrary.open(sqlitePath); }); - sqlite_open.open.overrideFor(sqlite_open.OperatingSystem.macOS, () { - return DynamicLibrary.open( - '/Users/simon/src/sqlite_async.dart/sqlite-autoconf-3480000/.libs/libsqlite3.dylib'); - }); final db = super.open(options); db.createFunction( From 9b4b3dfa152b9a7ed34002f7ccd039144c5f0683 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Mon, 27 Jan 2025 09:40:45 +0100 Subject: [PATCH 39/90] Fix typo --- packages/sqlite_async/lib/src/web/protocol.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/sqlite_async/lib/src/web/protocol.dart b/packages/sqlite_async/lib/src/web/protocol.dart index a20d4f5..cb3a5fd 100644 --- a/packages/sqlite_async/lib/src/web/protocol.dart +++ b/packages/sqlite_async/lib/src/web/protocol.dart @@ -26,7 +26,7 @@ extension type CustomDatabaseMessage._raw(JSObject _) implements JSObject { factory CustomDatabaseMessage(CustomDatabaseMessageKind kind, [String? sql, List parameters = const []]) { final rawSql = (sql ?? '').toJS; - // Serializing parameters this was is backwards-compatible with dartify() + // Serializing parameters this way is backwards-compatible with dartify() // on the other end, but a bit more efficient while also suppporting sound // communcation between dart2js workers and dart2wasm clients. // Older workers ignore the typeInfo, but that's not a problem. From 33750bebd362b257a513f9e8aed91d0c6c60a7a6 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Mon, 27 Jan 2025 12:20:23 +0100 Subject: [PATCH 40/90] Test serialized exceptions --- packages/sqlite_async/lib/src/web/database.dart | 7 ++++++- packages/sqlite_async/pubspec.yaml | 4 ---- packages/sqlite_async/test/basic_test.dart | 13 +++++++++++++ 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/packages/sqlite_async/lib/src/web/database.dart b/packages/sqlite_async/lib/src/web/database.dart index dc3a7af..04da846 100644 --- a/packages/sqlite_async/lib/src/web/database.dart +++ b/packages/sqlite_async/lib/src/web/database.dart @@ -311,9 +311,14 @@ Future wrapSqliteException(Future Function() callback) async { try { return await callback(); } on RemoteException catch (ex) { + if (ex.exception case final serializedCause?) { + throw serializedCause; + } + + // Older versions of package:sqlite_web reported SqliteExceptions as strings + // only. if (ex.toString().contains('SqliteException')) { RegExp regExp = RegExp(r'SqliteException\((\d+)\)'); - // The SQLite Web package wraps these in remote errors throw SqliteException( int.parse(regExp.firstMatch(ex.message)?.group(1) ?? '0'), ex.message); diff --git a/packages/sqlite_async/pubspec.yaml b/packages/sqlite_async/pubspec.yaml index 0a616b9..f6b539c 100644 --- a/packages/sqlite_async/pubspec.yaml +++ b/packages/sqlite_async/pubspec.yaml @@ -32,10 +32,6 @@ dev_dependencies: path: ^1.9.0 test_descriptor: ^2.0.2 -dependency_overrides: - sqlite3_web: - path: /Users/simon/src/sqlite3.dart/sqlite3_web - platforms: android: ios: diff --git a/packages/sqlite_async/test/basic_test.dart b/packages/sqlite_async/test/basic_test.dart index 83169fe..0aca7bc 100644 --- a/packages/sqlite_async/test/basic_test.dart +++ b/packages/sqlite_async/test/basic_test.dart @@ -175,6 +175,19 @@ void main() { ? 'Fails due to compiler bug, https://dartbug.com/59981' : null, ); + + test('reports exceptions as SqliteExceptions', () async { + final db = await testUtils.setupDatabase(path: path); + await expectLater( + db.get('SELECT invalid_statement;'), + throwsA( + isA() + .having((e) => e.causingStatement, 'causingStatement', + 'SELECT invalid_statement;') + .having((e) => e.extendedResultCode, 'extendedResultCode', 1), + ), + ); + }); }); } From 5b98ac156a5b50df8283b16db4a7dc5d64b27da1 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Mon, 27 Jan 2025 12:22:07 +0100 Subject: [PATCH 41/90] Extend lints dependency range --- packages/drift_sqlite_async/pubspec.yaml | 2 +- packages/sqlite_async/pubspec.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/drift_sqlite_async/pubspec.yaml b/packages/drift_sqlite_async/pubspec.yaml index 197f9b8..6093abf 100644 --- a/packages/drift_sqlite_async/pubspec.yaml +++ b/packages/drift_sqlite_async/pubspec.yaml @@ -21,7 +21,7 @@ dev_dependencies: build_runner: ^2.4.8 drift_dev: ">=2.19.0 <3.0.0" glob: ^2.1.2 - lints: ^5.1.1 + lints: ^5.0.0 sqlite3: ^2.4.0 test: ^1.25.2 test_api: ^0.7.0 diff --git a/packages/sqlite_async/pubspec.yaml b/packages/sqlite_async/pubspec.yaml index f6b539c..06c4246 100644 --- a/packages/sqlite_async/pubspec.yaml +++ b/packages/sqlite_async/pubspec.yaml @@ -21,7 +21,7 @@ dependencies: web: ^1.0.0 dev_dependencies: - lints: ^5.1.1 + lints: ^5.0.0 test: ^1.21.0 test_api: ^0.7.0 glob: ^2.1.1 From f626a2aaa116689ffba30ca8ab0c61d71359d4c4 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Mon, 27 Jan 2025 12:26:40 +0100 Subject: [PATCH 42/90] Fix two more analysis warnings --- packages/sqlite_async/test/close_test.dart | 2 ++ packages/sqlite_async/test/utils/native_test_utils.dart | 1 + 2 files changed, 3 insertions(+) diff --git a/packages/sqlite_async/test/close_test.dart b/packages/sqlite_async/test/close_test.dart index dcb3390..6a72f3d 100644 --- a/packages/sqlite_async/test/close_test.dart +++ b/packages/sqlite_async/test/close_test.dart @@ -1,4 +1,6 @@ @TestOn('!browser') +library; + import 'dart:io'; import 'package:sqlite_async/sqlite_async.dart'; diff --git a/packages/sqlite_async/test/utils/native_test_utils.dart b/packages/sqlite_async/test/utils/native_test_utils.dart index 31db65c..66bf57c 100644 --- a/packages/sqlite_async/test/utils/native_test_utils.dart +++ b/packages/sqlite_async/test/utils/native_test_utils.dart @@ -51,6 +51,7 @@ class TestSqliteOpenFactory extends TestDefaultSqliteOpenFactory { } class TestUtils extends AbstractTestUtils { + @override String dbPath() { return d.path('test.db'); } From 281d81cb893f1c6ed19d896ffc175c08ba9bd6be Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Mon, 27 Jan 2025 12:29:51 +0100 Subject: [PATCH 43/90] Fix typo in test option --- melos.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/melos.yaml b/melos.yaml index 875e7a5..bd27884 100644 --- a/melos.yaml +++ b/melos.yaml @@ -41,7 +41,7 @@ scripts: test: description: Run tests in a specific package. - run: dart test -p chrome,vm --compilers dart2js,dart2wasm + run: dart test -p chrome,vm --compiler dart2js,dart2wasm exec: concurrency: 1 packageFilters: From dbc501ba96cecd5999ac6f83cec9e4a433322f55 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Mon, 27 Jan 2025 12:41:47 +0100 Subject: [PATCH 44/90] Test with Dart stable, add changelogs --- .github/workflows/test.yaml | 10 +++++----- packages/drift_sqlite_async/CHANGELOG.md | 4 ++++ packages/drift_sqlite_async/pubspec.yaml | 2 +- packages/sqlite_async/CHANGELOG.md | 5 +++++ packages/sqlite_async/pubspec.yaml | 2 +- 5 files changed, 16 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 6f6cad4..c1bcd0a 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -35,19 +35,19 @@ jobs: include: - sqlite_version: "3440200" sqlite_url: "https://www.sqlite.org/2023/sqlite-autoconf-3440200.tar.gz" - dart_sdk: 3.5.0 + dart_sdk: stable - sqlite_version: "3430200" sqlite_url: "https://www.sqlite.org/2023/sqlite-autoconf-3430200.tar.gz" - dart_sdk: 3.5.0 + dart_sdk: stable - sqlite_version: "3420000" sqlite_url: "https://www.sqlite.org/2023/sqlite-autoconf-3420000.tar.gz" - dart_sdk: 3.5.0 + dart_sdk: stable - sqlite_version: "3410100" sqlite_url: "https://www.sqlite.org/2023/sqlite-autoconf-3410100.tar.gz" - dart_sdk: 3.5.0 + dart_sdk: stable - sqlite_version: "3380000" sqlite_url: "https://www.sqlite.org/2022/sqlite-autoconf-3380000.tar.gz" - dart_sdk: 3.5.0 + dart_sdk: stable steps: - uses: actions/checkout@v3 - uses: dart-lang/setup-dart@v1 diff --git a/packages/drift_sqlite_async/CHANGELOG.md b/packages/drift_sqlite_async/CHANGELOG.md index 4c23e85..d39db6f 100644 --- a/packages/drift_sqlite_async/CHANGELOG.md +++ b/packages/drift_sqlite_async/CHANGELOG.md @@ -1,3 +1,7 @@ +## 0.2.1 + +- Fix lints. + ## 0.2.0 - Automatically run Drift migrations diff --git a/packages/drift_sqlite_async/pubspec.yaml b/packages/drift_sqlite_async/pubspec.yaml index 6093abf..98eb766 100644 --- a/packages/drift_sqlite_async/pubspec.yaml +++ b/packages/drift_sqlite_async/pubspec.yaml @@ -1,5 +1,5 @@ name: drift_sqlite_async -version: 0.2.0 +version: 0.2.1 homepage: https://github.com/powersync-ja/sqlite_async.dart repository: https://github.com/powersync-ja/sqlite_async.dart description: Use Drift with a sqlite_async database, allowing both to be used in the same application. diff --git a/packages/sqlite_async/CHANGELOG.md b/packages/sqlite_async/CHANGELOG.md index b49c6ab..ea77953 100644 --- a/packages/sqlite_async/CHANGELOG.md +++ b/packages/sqlite_async/CHANGELOG.md @@ -1,3 +1,8 @@ +## 0.11.2 + +- Support latest version of `package:sqlite3_web`. +- Support `dart2wasm`. + ## 0.11.1 - Remove remaining `dart:js_util` imports in favor of new interop APIs. diff --git a/packages/sqlite_async/pubspec.yaml b/packages/sqlite_async/pubspec.yaml index 06c4246..e4ea5ce 100644 --- a/packages/sqlite_async/pubspec.yaml +++ b/packages/sqlite_async/pubspec.yaml @@ -1,6 +1,6 @@ name: sqlite_async description: High-performance asynchronous interface for SQLite on Dart and Flutter. -version: 0.11.1 +version: 0.11.2 repository: https://github.com/powersync-ja/sqlite_async.dart environment: sdk: ">=3.5.0 <4.0.0" From 7ff4bfee8db1f0e111134738a73cfe4c4db10c4e Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Mon, 3 Feb 2025 10:40:54 +0100 Subject: [PATCH 45/90] Support build_web_compilers --- .github/workflows/test.yaml | 1 + melos.yaml | 8 + packages/sqlite_async/CHANGELOG.md | 4 + .../lib/src/common/port_channel.dart | 354 +----------------- .../lib/src/common/port_channel_native.dart | 352 +++++++++++++++++ .../lib/src/common/port_channel_stub.dart | 149 ++++++++ packages/sqlite_async/pubspec.yaml | 5 +- 7 files changed, 520 insertions(+), 353 deletions(-) create mode 100644 packages/sqlite_async/lib/src/common/port_channel_native.dart create mode 100644 packages/sqlite_async/lib/src/common/port_channel_stub.dart diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index c1bcd0a..cb1814b 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -68,3 +68,4 @@ jobs: run: | export LD_LIBRARY_PATH=$(pwd)/sqlite-autoconf-${{ matrix.sqlite_version }}/.libs melos test + melos test_build diff --git a/melos.yaml b/melos.yaml index bd27884..4555e43 100644 --- a/melos.yaml +++ b/melos.yaml @@ -51,3 +51,11 @@ scripts: # as they could change the behaviour of how tests filter packages. env: MELOS_TEST: true + + test_build: + description: Runs tests with build_test + run: dart run build_runner test -- -p chrome + exec: + concurrency: 1 + packageFilters: + dependsOn: build_test diff --git a/packages/sqlite_async/CHANGELOG.md b/packages/sqlite_async/CHANGELOG.md index ea77953..f3395b1 100644 --- a/packages/sqlite_async/CHANGELOG.md +++ b/packages/sqlite_async/CHANGELOG.md @@ -1,3 +1,7 @@ +## 0.11.3 + +- Support being compiled with `package:build_web_compilers`. + ## 0.11.2 - Support latest version of `package:sqlite3_web`. diff --git a/packages/sqlite_async/lib/src/common/port_channel.dart b/packages/sqlite_async/lib/src/common/port_channel.dart index 8b05feb..fc5e69c 100644 --- a/packages/sqlite_async/lib/src/common/port_channel.dart +++ b/packages/sqlite_async/lib/src/common/port_channel.dart @@ -1,352 +1,2 @@ -import 'dart:async'; -import 'dart:collection'; -import 'dart:isolate'; - -abstract class PortClient { - Future post(Object message); - void fire(Object message); - - factory PortClient.parent() { - return ParentPortClient(); - } - - factory PortClient.child(SendPort upstream) { - return ChildPortClient(upstream); - } -} - -class ParentPortClient implements PortClient { - late Future sendPortFuture; - SendPort? sendPort; - final ReceivePort _receivePort = ReceivePort(); - final ReceivePort _errorPort = ReceivePort(); - bool closed = false; - Object? _closeError; - String? _isolateDebugName; - int _nextId = 1; - - Map> handlers = HashMap(); - - ParentPortClient() { - final initCompleter = Completer.sync(); - sendPortFuture = initCompleter.future; - sendPortFuture.then((value) { - sendPort = value; - }); - _receivePort.listen((message) { - if (message is _InitMessage) { - assert(!initCompleter.isCompleted); - initCompleter.complete(message.port); - } else if (message is _PortChannelResult) { - final handler = handlers.remove(message.requestId); - assert(handler != null); - if (message.success) { - handler!.complete(message.result); - } else { - handler!.completeError(message.error, message.stackTrace); - } - } else if (message == _closeMessage) { - close(); - } - }, onError: (e) { - if (!initCompleter.isCompleted) { - initCompleter.completeError(e); - } - - close(); - }, onDone: () { - if (!initCompleter.isCompleted) { - initCompleter.completeError(ClosedException()); - } - close(); - }); - _errorPort.listen((message) { - final [error, stackTraceString] = message; - final stackTrace = stackTraceString == null - ? null - : StackTrace.fromString(stackTraceString); - if (!initCompleter.isCompleted) { - initCompleter.completeError(error, stackTrace); - } - _close(IsolateError(cause: error, isolateDebugName: _isolateDebugName), - stackTrace); - }); - } - - Future get ready async { - await sendPortFuture; - } - - void _cancelAll(Object error, [StackTrace? stackTrace]) { - var handlers = this.handlers; - this.handlers = {}; - for (var message in handlers.values) { - message.completeError(error, stackTrace); - } - } - - @override - Future post(Object message) async { - if (closed) { - throw _closeError ?? const ClosedException(); - } - var completer = Completer.sync(); - var id = _nextId++; - handlers[id] = completer; - final port = sendPort ?? await sendPortFuture; - port.send(_RequestMessage(id, message, null)); - return await completer.future; - } - - @override - void fire(Object message) async { - if (closed) { - throw _closeError ?? ClosedException(); - } - final port = sendPort ?? await sendPortFuture; - port.send(_FireMessage(message)); - } - - RequestPortServer server() { - return RequestPortServer(_receivePort.sendPort); - } - - void _close([Object? error, StackTrace? stackTrace]) { - if (!closed) { - closed = true; - - _receivePort.close(); - _errorPort.close(); - if (error == null) { - _cancelAll(const ClosedException()); - } else { - _closeError = error; - _cancelAll(error, stackTrace); - } - } - } - - void close() { - _close(); - } - - tieToIsolate(Isolate isolate) { - _isolateDebugName = isolate.debugName; - isolate.addErrorListener(_errorPort.sendPort); - isolate.addOnExitListener(_receivePort.sendPort, response: _closeMessage); - } -} - -class SerializedPortClient { - final SendPort sendPort; - - SerializedPortClient(this.sendPort); - - ChildPortClient open() { - return ChildPortClient(sendPort); - } -} - -class ChildPortClient implements PortClient { - final SendPort sendPort; - final ReceivePort receivePort = ReceivePort(); - int _nextId = 1; - bool closed = false; - - final Map> handlers = HashMap(); - - ChildPortClient(this.sendPort) { - receivePort.listen((message) { - if (message is _PortChannelResult) { - final handler = handlers.remove(message.requestId); - assert(handler != null); - if (message.success) { - handler!.complete(message.result); - } else { - handler!.completeError(message.error, message.stackTrace); - } - } - }); - } - - @override - Future post(Object message) async { - if (closed) { - throw const ClosedException(); - } - var completer = Completer.sync(); - var id = _nextId++; - handlers[id] = completer; - sendPort.send(_RequestMessage(id, message, receivePort.sendPort)); - return await completer.future; - } - - @override - void fire(Object message) { - if (closed) { - throw ClosedException(); - } - sendPort.send(_FireMessage(message)); - } - - void _cancelAll(Object error) { - var handlers = HashMap>.from(this.handlers); - this.handlers.clear(); - for (var message in handlers.values) { - message.completeError(error); - } - } - - void close() { - closed = true; - _cancelAll(const ClosedException()); - receivePort.close(); - } -} - -class RequestPortServer { - final SendPort port; - - RequestPortServer(this.port); - - open(Future Function(Object? message) handle) { - return PortServer.forSendPort(port, handle); - } -} - -class PortServer { - final ReceivePort _receivePort = ReceivePort(); - final Future Function(Object? message) handle; - final SendPort? replyPort; - - PortServer(this.handle) : replyPort = null { - _init(); - } - - PortServer.forSendPort(SendPort port, this.handle) : replyPort = port { - port.send(_InitMessage(_receivePort.sendPort)); - _init(); - } - - SendPort get sendPort { - return _receivePort.sendPort; - } - - SerializedPortClient client() { - return SerializedPortClient(sendPort); - } - - void close() { - _receivePort.close(); - } - - void _init() { - _receivePort.listen((request) async { - if (request is _FireMessage) { - handle(request.message); - } else if (request is _RequestMessage) { - if (request.id == 0) { - // Fire and forget - handle(request.message); - } else { - final replyPort = request.reply ?? this.replyPort; - try { - var result = await handle(request.message); - replyPort!.send(_PortChannelResult.success(request.id, result)); - } catch (e, stacktrace) { - replyPort! - .send(_PortChannelResult.error(request.id, e, stacktrace)); - } - } - } - }); - } -} - -const _closeMessage = '_Close'; - -class _InitMessage { - final SendPort port; - - _InitMessage(this.port); -} - -class _FireMessage { - final Object message; - - const _FireMessage(this.message); -} - -class _RequestMessage { - final int id; - final Object message; - final SendPort? reply; - - _RequestMessage(this.id, this.message, this.reply); -} - -class ClosedException implements Exception { - const ClosedException(); - - @override - String toString() { - return 'ClosedException'; - } -} - -class IsolateError extends Error { - final Object cause; - final String? isolateDebugName; - - IsolateError({required this.cause, this.isolateDebugName}); - - @override - String toString() { - if (isolateDebugName != null) { - return 'IsolateError in $isolateDebugName: $cause'; - } else { - return 'IsolateError: $cause'; - } - } -} - -class _PortChannelResult { - final int requestId; - final bool success; - final T? _result; - final Object? _error; - final StackTrace? stackTrace; - - const _PortChannelResult.success(this.requestId, T result) - : success = true, - _error = null, - stackTrace = null, - _result = result; - const _PortChannelResult.error(this.requestId, Object error, - [this.stackTrace]) - : success = false, - _result = null, - _error = error; - - T get value { - if (success) { - return _result as T; - } else { - if (_error != null && stackTrace != null) { - Error.throwWithStackTrace(_error, stackTrace!); - } else { - throw _error!; - } - } - } - - T get result { - assert(success); - return _result as T; - } - - Object get error { - assert(!success); - return _error!; - } -} +export 'port_channel_native.dart' + if (dart.library.js_interop) 'port_channel_stub.dart'; diff --git a/packages/sqlite_async/lib/src/common/port_channel_native.dart b/packages/sqlite_async/lib/src/common/port_channel_native.dart new file mode 100644 index 0000000..8b05feb --- /dev/null +++ b/packages/sqlite_async/lib/src/common/port_channel_native.dart @@ -0,0 +1,352 @@ +import 'dart:async'; +import 'dart:collection'; +import 'dart:isolate'; + +abstract class PortClient { + Future post(Object message); + void fire(Object message); + + factory PortClient.parent() { + return ParentPortClient(); + } + + factory PortClient.child(SendPort upstream) { + return ChildPortClient(upstream); + } +} + +class ParentPortClient implements PortClient { + late Future sendPortFuture; + SendPort? sendPort; + final ReceivePort _receivePort = ReceivePort(); + final ReceivePort _errorPort = ReceivePort(); + bool closed = false; + Object? _closeError; + String? _isolateDebugName; + int _nextId = 1; + + Map> handlers = HashMap(); + + ParentPortClient() { + final initCompleter = Completer.sync(); + sendPortFuture = initCompleter.future; + sendPortFuture.then((value) { + sendPort = value; + }); + _receivePort.listen((message) { + if (message is _InitMessage) { + assert(!initCompleter.isCompleted); + initCompleter.complete(message.port); + } else if (message is _PortChannelResult) { + final handler = handlers.remove(message.requestId); + assert(handler != null); + if (message.success) { + handler!.complete(message.result); + } else { + handler!.completeError(message.error, message.stackTrace); + } + } else if (message == _closeMessage) { + close(); + } + }, onError: (e) { + if (!initCompleter.isCompleted) { + initCompleter.completeError(e); + } + + close(); + }, onDone: () { + if (!initCompleter.isCompleted) { + initCompleter.completeError(ClosedException()); + } + close(); + }); + _errorPort.listen((message) { + final [error, stackTraceString] = message; + final stackTrace = stackTraceString == null + ? null + : StackTrace.fromString(stackTraceString); + if (!initCompleter.isCompleted) { + initCompleter.completeError(error, stackTrace); + } + _close(IsolateError(cause: error, isolateDebugName: _isolateDebugName), + stackTrace); + }); + } + + Future get ready async { + await sendPortFuture; + } + + void _cancelAll(Object error, [StackTrace? stackTrace]) { + var handlers = this.handlers; + this.handlers = {}; + for (var message in handlers.values) { + message.completeError(error, stackTrace); + } + } + + @override + Future post(Object message) async { + if (closed) { + throw _closeError ?? const ClosedException(); + } + var completer = Completer.sync(); + var id = _nextId++; + handlers[id] = completer; + final port = sendPort ?? await sendPortFuture; + port.send(_RequestMessage(id, message, null)); + return await completer.future; + } + + @override + void fire(Object message) async { + if (closed) { + throw _closeError ?? ClosedException(); + } + final port = sendPort ?? await sendPortFuture; + port.send(_FireMessage(message)); + } + + RequestPortServer server() { + return RequestPortServer(_receivePort.sendPort); + } + + void _close([Object? error, StackTrace? stackTrace]) { + if (!closed) { + closed = true; + + _receivePort.close(); + _errorPort.close(); + if (error == null) { + _cancelAll(const ClosedException()); + } else { + _closeError = error; + _cancelAll(error, stackTrace); + } + } + } + + void close() { + _close(); + } + + tieToIsolate(Isolate isolate) { + _isolateDebugName = isolate.debugName; + isolate.addErrorListener(_errorPort.sendPort); + isolate.addOnExitListener(_receivePort.sendPort, response: _closeMessage); + } +} + +class SerializedPortClient { + final SendPort sendPort; + + SerializedPortClient(this.sendPort); + + ChildPortClient open() { + return ChildPortClient(sendPort); + } +} + +class ChildPortClient implements PortClient { + final SendPort sendPort; + final ReceivePort receivePort = ReceivePort(); + int _nextId = 1; + bool closed = false; + + final Map> handlers = HashMap(); + + ChildPortClient(this.sendPort) { + receivePort.listen((message) { + if (message is _PortChannelResult) { + final handler = handlers.remove(message.requestId); + assert(handler != null); + if (message.success) { + handler!.complete(message.result); + } else { + handler!.completeError(message.error, message.stackTrace); + } + } + }); + } + + @override + Future post(Object message) async { + if (closed) { + throw const ClosedException(); + } + var completer = Completer.sync(); + var id = _nextId++; + handlers[id] = completer; + sendPort.send(_RequestMessage(id, message, receivePort.sendPort)); + return await completer.future; + } + + @override + void fire(Object message) { + if (closed) { + throw ClosedException(); + } + sendPort.send(_FireMessage(message)); + } + + void _cancelAll(Object error) { + var handlers = HashMap>.from(this.handlers); + this.handlers.clear(); + for (var message in handlers.values) { + message.completeError(error); + } + } + + void close() { + closed = true; + _cancelAll(const ClosedException()); + receivePort.close(); + } +} + +class RequestPortServer { + final SendPort port; + + RequestPortServer(this.port); + + open(Future Function(Object? message) handle) { + return PortServer.forSendPort(port, handle); + } +} + +class PortServer { + final ReceivePort _receivePort = ReceivePort(); + final Future Function(Object? message) handle; + final SendPort? replyPort; + + PortServer(this.handle) : replyPort = null { + _init(); + } + + PortServer.forSendPort(SendPort port, this.handle) : replyPort = port { + port.send(_InitMessage(_receivePort.sendPort)); + _init(); + } + + SendPort get sendPort { + return _receivePort.sendPort; + } + + SerializedPortClient client() { + return SerializedPortClient(sendPort); + } + + void close() { + _receivePort.close(); + } + + void _init() { + _receivePort.listen((request) async { + if (request is _FireMessage) { + handle(request.message); + } else if (request is _RequestMessage) { + if (request.id == 0) { + // Fire and forget + handle(request.message); + } else { + final replyPort = request.reply ?? this.replyPort; + try { + var result = await handle(request.message); + replyPort!.send(_PortChannelResult.success(request.id, result)); + } catch (e, stacktrace) { + replyPort! + .send(_PortChannelResult.error(request.id, e, stacktrace)); + } + } + } + }); + } +} + +const _closeMessage = '_Close'; + +class _InitMessage { + final SendPort port; + + _InitMessage(this.port); +} + +class _FireMessage { + final Object message; + + const _FireMessage(this.message); +} + +class _RequestMessage { + final int id; + final Object message; + final SendPort? reply; + + _RequestMessage(this.id, this.message, this.reply); +} + +class ClosedException implements Exception { + const ClosedException(); + + @override + String toString() { + return 'ClosedException'; + } +} + +class IsolateError extends Error { + final Object cause; + final String? isolateDebugName; + + IsolateError({required this.cause, this.isolateDebugName}); + + @override + String toString() { + if (isolateDebugName != null) { + return 'IsolateError in $isolateDebugName: $cause'; + } else { + return 'IsolateError: $cause'; + } + } +} + +class _PortChannelResult { + final int requestId; + final bool success; + final T? _result; + final Object? _error; + final StackTrace? stackTrace; + + const _PortChannelResult.success(this.requestId, T result) + : success = true, + _error = null, + stackTrace = null, + _result = result; + const _PortChannelResult.error(this.requestId, Object error, + [this.stackTrace]) + : success = false, + _result = null, + _error = error; + + T get value { + if (success) { + return _result as T; + } else { + if (_error != null && stackTrace != null) { + Error.throwWithStackTrace(_error, stackTrace!); + } else { + throw _error!; + } + } + } + + T get result { + assert(success); + return _result as T; + } + + Object get error { + assert(!success); + return _error!; + } +} diff --git a/packages/sqlite_async/lib/src/common/port_channel_stub.dart b/packages/sqlite_async/lib/src/common/port_channel_stub.dart new file mode 100644 index 0000000..6c6e5cc --- /dev/null +++ b/packages/sqlite_async/lib/src/common/port_channel_stub.dart @@ -0,0 +1,149 @@ +import 'dart:async'; +import 'dart:collection'; + +typedef SendPort = Never; +typedef ReceivePort = Never; +typedef Isolate = Never; + +Never _stub() { + throw UnsupportedError('Isolates are not supported on this platform'); +} + +abstract class PortClient { + Future post(Object message); + void fire(Object message); + + factory PortClient.parent() { + return ParentPortClient(); + } + + factory PortClient.child(SendPort upstream) { + return ChildPortClient(upstream); + } +} + +class ParentPortClient implements PortClient { + late Future sendPortFuture; + SendPort? sendPort; + bool closed = false; + + Map> handlers = HashMap(); + + ParentPortClient(); + + Future get ready async { + await sendPortFuture; + } + + @override + Future post(Object message) async { + _stub(); + } + + @override + void fire(Object message) async { + _stub(); + } + + RequestPortServer server() { + _stub(); + } + + void close() { + _stub(); + } + + tieToIsolate(Isolate isolate) { + _stub(); + } +} + +class SerializedPortClient { + final SendPort sendPort; + + SerializedPortClient(this.sendPort); + + ChildPortClient open() { + return ChildPortClient(sendPort); + } +} + +class ChildPortClient implements PortClient { + final SendPort sendPort; + ReceivePort get receivePort => _stub(); + bool closed = false; + + final Map> handlers = HashMap(); + + ChildPortClient(this.sendPort); + + @override + Future post(Object message) async { + _stub(); + } + + @override + void fire(Object message) { + _stub(); + } + + void close() { + _stub(); + } +} + +class RequestPortServer { + final SendPort port; + + RequestPortServer(this.port); + + open(Future Function(Object? message) handle) { + _stub(); + } +} + +class PortServer { + final Future Function(Object? message) handle; + final SendPort? replyPort; + + PortServer(this.handle) : replyPort = null; + + PortServer.forSendPort(SendPort port, this.handle) : replyPort = port; + + SendPort get sendPort { + _stub(); + } + + SerializedPortClient client() { + return SerializedPortClient(sendPort); + } + + void close() { + _stub(); + } +} + +class ClosedException implements Exception { + const ClosedException(); + + @override + String toString() { + return 'ClosedException'; + } +} + +class IsolateError extends Error { + final Object cause; + final String? isolateDebugName; + + IsolateError({required this.cause, this.isolateDebugName}); + + @override + String toString() { + if (isolateDebugName != null) { + return 'IsolateError in $isolateDebugName: $cause'; + } else { + return 'IsolateError: $cause'; + } + } +} diff --git a/packages/sqlite_async/pubspec.yaml b/packages/sqlite_async/pubspec.yaml index e4ea5ce..8264912 100644 --- a/packages/sqlite_async/pubspec.yaml +++ b/packages/sqlite_async/pubspec.yaml @@ -1,6 +1,6 @@ name: sqlite_async description: High-performance asynchronous interface for SQLite on Dart and Flutter. -version: 0.11.2 +version: 0.11.3 repository: https://github.com/powersync-ja/sqlite_async.dart environment: sdk: ">=3.5.0 <4.0.0" @@ -21,6 +21,9 @@ dependencies: web: ^1.0.0 dev_dependencies: + build_runner: ^2.4.14 + build_web_compilers: ^4.1.1 + build_test: ^2.2.3 lints: ^5.0.0 test: ^1.21.0 test_api: ^0.7.0 From 1d8de884ef263ec6e9b5f358d2d13b922fba8bc4 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Tue, 4 Feb 2025 15:51:51 +0100 Subject: [PATCH 46/90] Allow wrapping `CommonDatabase` directly --- packages/sqlite_async/CHANGELOG.md | 7 +++ .../lib/src/common/sqlite_database.dart | 21 +++++++ .../src/impl/single_connection_database.dart | 60 +++++++++++++++++++ .../lib/src/sqlite_connection.dart | 23 +++++++ packages/sqlite_async/pubspec.yaml | 2 +- packages/sqlite_async/test/basic_test.dart | 21 +++++++ .../test/utils/abstract_test_utils.dart | 5 ++ .../test/utils/native_test_utils.dart | 15 ++++- .../test/utils/web_test_utils.dart | 24 +++++++- 9 files changed, 172 insertions(+), 6 deletions(-) create mode 100644 packages/sqlite_async/lib/src/impl/single_connection_database.dart diff --git a/packages/sqlite_async/CHANGELOG.md b/packages/sqlite_async/CHANGELOG.md index f3395b1..0038ad2 100644 --- a/packages/sqlite_async/CHANGELOG.md +++ b/packages/sqlite_async/CHANGELOG.md @@ -1,3 +1,10 @@ +## 0.11.4 + +- Add `SqliteConnection.synchronousWrapper` and `SqliteDatabase.singleConnection`. + Together, these can be used to wrap raw `CommonDatabase` instances from `package:sqlite3` + as a `Database` (without an automated worker or isolate setup). This can be useful in tests + where synchronous access to the underlying database is convenient. + ## 0.11.3 - Support being compiled with `package:build_web_compilers`. diff --git a/packages/sqlite_async/lib/src/common/sqlite_database.dart b/packages/sqlite_async/lib/src/common/sqlite_database.dart index f8e0be5..3cb12bb 100644 --- a/packages/sqlite_async/lib/src/common/sqlite_database.dart +++ b/packages/sqlite_async/lib/src/common/sqlite_database.dart @@ -3,6 +3,7 @@ import 'dart:async'; import 'package:meta/meta.dart'; import 'package:sqlite_async/src/common/abstract_open_factory.dart'; import 'package:sqlite_async/src/common/isolate_connection_factory.dart'; +import 'package:sqlite_async/src/impl/single_connection_database.dart'; import 'package:sqlite_async/src/impl/sqlite_database_impl.dart'; import 'package:sqlite_async/src/sqlite_options.dart'; import 'package:sqlite_async/src/sqlite_queries.dart'; @@ -82,4 +83,24 @@ abstract class SqliteDatabase {int maxReaders = SqliteDatabase.defaultMaxReaders}) { return SqliteDatabaseImpl.withFactory(openFactory, maxReaders: maxReaders); } + + /// Opens a [SqliteDatabase] that only wraps an underlying connection. + /// + /// This function may be useful in some instances like tests, but should not + /// typically be used by applications. Compared to the other ways to open + /// databases, it has the following downsides: + /// + /// 1. No connection pool / concurrent readers for native databases. + /// 2. No reliable update notifications on the web. + /// 3. There is no reliable transaction management in Dart, and opening the + /// same database with [SqliteDatabase.singleConnection] multiple times + /// may cause "database is locked" errors. + /// + /// Together with [SqliteConnection.synchronousWrapper], this can be used to + /// open in-memory databases (e.g. via [SqliteOpenFactory.open]). That + /// bypasses most convenience features, but may still be useful for + /// short-lived databases used in tests. + factory SqliteDatabase.singleConnection(SqliteConnection connection) { + return SingleConnectionDatabase(connection); + } } diff --git a/packages/sqlite_async/lib/src/impl/single_connection_database.dart b/packages/sqlite_async/lib/src/impl/single_connection_database.dart new file mode 100644 index 0000000..4cd3144 --- /dev/null +++ b/packages/sqlite_async/lib/src/impl/single_connection_database.dart @@ -0,0 +1,60 @@ +import 'package:sqlite3/common.dart'; +import 'package:sqlite_async/sqlite_async.dart'; + +/// A database implementation that delegates everything to a single connection. +/// +/// This doesn't provide an automatic connection pool or the web worker +/// management, but it can still be useful in cases like unit tests where those +/// features might not be necessary. Since only a single sqlite connection is +/// used internally, this also allows using in-memory databases. +final class SingleConnectionDatabase + with SqliteQueries, SqliteDatabaseMixin + implements SqliteDatabase { + final SqliteConnection connection; + + SingleConnectionDatabase(this.connection); + + @override + Future close() => connection.close(); + + @override + bool get closed => connection.closed; + + @override + Future getAutoCommit() => connection.getAutoCommit(); + + @override + Future get isInitialized => Future.value(); + + @override + IsolateConnectionFactory isolateConnectionFactory() { + throw UnsupportedError( + "SqliteDatabase.singleConnection instances can't be used across " + 'isolates.'); + } + + @override + int get maxReaders => 1; + + @override + AbstractDefaultSqliteOpenFactory get openFactory => + throw UnimplementedError(); + + @override + Future readLock(Future Function(SqliteReadContext tx) callback, + {Duration? lockTimeout, String? debugContext}) { + return connection.readLock(callback, + lockTimeout: lockTimeout, debugContext: debugContext); + } + + @override + Stream get updates => + connection.updates ?? const Stream.empty(); + + @override + Future writeLock(Future Function(SqliteWriteContext tx) callback, + {Duration? lockTimeout, String? debugContext}) { + return connection.writeLock(callback, + lockTimeout: lockTimeout, debugContext: debugContext); + } +} diff --git a/packages/sqlite_async/lib/src/sqlite_connection.dart b/packages/sqlite_async/lib/src/sqlite_connection.dart index f1b721a..15f4f6a 100644 --- a/packages/sqlite_async/lib/src/sqlite_connection.dart +++ b/packages/sqlite_async/lib/src/sqlite_connection.dart @@ -1,8 +1,12 @@ import 'dart:async'; import 'package:sqlite3/common.dart' as sqlite; +import 'package:sqlite_async/mutex.dart'; +import 'package:sqlite_async/sqlite3_common.dart'; import 'package:sqlite_async/src/update_notification.dart'; +import 'common/connection/sync_sqlite_connection.dart'; + /// Abstract class representing calls available in a read-only or read-write context. abstract class SqliteReadContext { /// Execute a read-only (SELECT) query and return the results. @@ -74,7 +78,26 @@ abstract class SqliteWriteContext extends SqliteReadContext { } /// Abstract class representing a connection to the SQLite database. +/// +/// This package typically pools multiple [SqliteConnection] instances into a +/// managed [SqliteDatabase] automatically. abstract class SqliteConnection extends SqliteWriteContext { + /// Default constructor for subclasses. + SqliteConnection(); + + /// Creates a [SqliteConnection] instance that wraps a raw [CommonDatabase] + /// from the `sqlite3` package. + /// + /// Users should not typically create connections manually at all. Instead, + /// open a [SqliteDatabase] through a factory. In special scenarios where it + /// may be easier to wrap a [raw] databases (like unit tests), this method + /// may be used as an escape hatch for the asynchronous wrappers provided by + /// this package. + factory SqliteConnection.synchronousWrapper(CommonDatabase raw, + {Mutex? mutex}) { + return SyncSqliteConnection(raw, mutex ?? Mutex()); + } + /// Reports table change update notifications Stream? get updates; diff --git a/packages/sqlite_async/pubspec.yaml b/packages/sqlite_async/pubspec.yaml index 8264912..d333166 100644 --- a/packages/sqlite_async/pubspec.yaml +++ b/packages/sqlite_async/pubspec.yaml @@ -1,6 +1,6 @@ name: sqlite_async description: High-performance asynchronous interface for SQLite on Dart and Flutter. -version: 0.11.3 +version: 0.11.4 repository: https://github.com/powersync-ja/sqlite_async.dart environment: sdk: ">=3.5.0 <4.0.0" diff --git a/packages/sqlite_async/test/basic_test.dart b/packages/sqlite_async/test/basic_test.dart index 0aca7bc..6ee038a 100644 --- a/packages/sqlite_async/test/basic_test.dart +++ b/packages/sqlite_async/test/basic_test.dart @@ -188,6 +188,27 @@ void main() { ), ); }); + + test('can use raw database instance', () async { + final factory = await testUtils.testFactory(); + final raw = await factory.openDatabaseForSingleConnection(); + // Creating a fuction ensures that this database is actually used - if + // a connection were set up in a background isolate, it wouldn't have this + // function. + raw.createFunction( + functionName: 'my_function', function: (args) => 'test'); + + final db = SqliteDatabase.singleConnection( + SqliteConnection.synchronousWrapper(raw)); + await createTables(db); + + expect(db.updates, emits(UpdateNotification({'test_data'}))); + await db + .execute('INSERT INTO test_data(description) VALUES (my_function())'); + + expect(await db.get('SELECT description FROM test_data'), + {'description': 'test'}); + }); }); } diff --git a/packages/sqlite_async/test/utils/abstract_test_utils.dart b/packages/sqlite_async/test/utils/abstract_test_utils.dart index e787a45..b388c4d 100644 --- a/packages/sqlite_async/test/utils/abstract_test_utils.dart +++ b/packages/sqlite_async/test/utils/abstract_test_utils.dart @@ -1,3 +1,4 @@ +import 'package:sqlite_async/sqlite3_common.dart'; import 'package:sqlite_async/sqlite_async.dart'; class TestDefaultSqliteOpenFactory extends DefaultSqliteOpenFactory { @@ -5,6 +6,10 @@ class TestDefaultSqliteOpenFactory extends DefaultSqliteOpenFactory { TestDefaultSqliteOpenFactory( {required super.path, super.sqliteOptions, this.sqlitePath = ''}); + + Future openDatabaseForSingleConnection() async { + return openDB(SqliteOpenOptions(primaryConnection: true, readOnly: false)); + } } abstract class AbstractTestUtils { diff --git a/packages/sqlite_async/test/utils/native_test_utils.dart b/packages/sqlite_async/test/utils/native_test_utils.dart index 66bf57c..945529d 100644 --- a/packages/sqlite_async/test/utils/native_test_utils.dart +++ b/packages/sqlite_async/test/utils/native_test_utils.dart @@ -5,6 +5,7 @@ import 'dart:isolate'; import 'package:glob/glob.dart'; import 'package:glob/list_local_fs.dart'; +import 'package:sqlite_async/sqlite3.dart'; import 'package:sqlite_async/sqlite3_common.dart'; import 'package:sqlite_async/sqlite_async.dart'; import 'package:sqlite3/open.dart' as sqlite_open; @@ -21,11 +22,15 @@ class TestSqliteOpenFactory extends TestDefaultSqliteOpenFactory { super.sqlitePath = defaultSqlitePath, initStatements}); - @override - CommonDatabase open(SqliteOpenOptions options) { + void _applyOpenOverrides() { sqlite_open.open.overrideFor(sqlite_open.OperatingSystem.linux, () { return DynamicLibrary.open(sqlitePath); }); + } + + @override + CommonDatabase open(SqliteOpenOptions options) { + _applyOpenOverrides(); final db = super.open(options); db.createFunction( @@ -48,6 +53,12 @@ class TestSqliteOpenFactory extends TestDefaultSqliteOpenFactory { return db; } + + @override + Future openDatabaseForSingleConnection() async { + _applyOpenOverrides(); + return sqlite3.openInMemory(); + } } class TestUtils extends AbstractTestUtils { diff --git a/packages/sqlite_async/test/utils/web_test_utils.dart b/packages/sqlite_async/test/utils/web_test_utils.dart index ac718f2..33f7d64 100644 --- a/packages/sqlite_async/test/utils/web_test_utils.dart +++ b/packages/sqlite_async/test/utils/web_test_utils.dart @@ -2,6 +2,7 @@ import 'dart:async'; import 'dart:js_interop'; import 'dart:math'; +import 'package:sqlite_async/sqlite3_wasm.dart'; import 'package:sqlite_async/sqlite_async.dart'; import 'package:test/test.dart'; import 'package:web/web.dart' show Blob, BlobPart, BlobPropertyBag; @@ -12,6 +13,20 @@ external String _createObjectURL(Blob blob); String? _dbPath; +class TestSqliteOpenFactory extends TestDefaultSqliteOpenFactory { + TestSqliteOpenFactory( + {required super.path, super.sqliteOptions, super.sqlitePath = ''}); + + @override + Future openDatabaseForSingleConnection() async { + final sqlite = await WasmSqlite3.loadFromUrl( + Uri.parse(sqliteOptions.webSqliteOptions.wasmUri)); + sqlite.registerVirtualFileSystem(InMemoryFileSystem(), makeDefault: true); + + return sqlite.openInMemory(); + } +} + class TestUtils extends AbstractTestUtils { late Future _isInitialized; late final SqliteOptions webOptions; @@ -57,12 +72,15 @@ class TestUtils extends AbstractTestUtils { @override Future testFactory( {String? path, - String? sqlitePath, + String sqlitePath = '', List initStatements = const [], SqliteOptions options = const SqliteOptions.defaults()}) async { await _isInitialized; - return super.testFactory( - path: path, options: webOptions, initStatements: initStatements); + return TestSqliteOpenFactory( + path: path ?? dbPath(), + sqlitePath: sqlitePath, + sqliteOptions: webOptions, + ); } @override From 996beeb6971d85088f6234fe558b3ecb161d9760 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Wed, 5 Feb 2025 12:57:45 +0100 Subject: [PATCH 47/90] Add tests for stream throttling --- .../lib/src/update_notification.dart | 25 ++-- packages/sqlite_async/pubspec.yaml | 1 + .../test/update_notification_test.dart | 134 ++++++++++++++++++ 3 files changed, 148 insertions(+), 12 deletions(-) create mode 100644 packages/sqlite_async/test/update_notification_test.dart diff --git a/packages/sqlite_async/lib/src/update_notification.dart b/packages/sqlite_async/lib/src/update_notification.dart index 8141df9..7832e32 100644 --- a/packages/sqlite_async/lib/src/update_notification.dart +++ b/packages/sqlite_async/lib/src/update_notification.dart @@ -62,12 +62,8 @@ class UpdateNotification { static StreamTransformer filterTablesTransformer(Iterable tables) { Set normalized = {for (var table in tables) table.toLowerCase()}; - return StreamTransformer.fromHandlers(handleData: (data, sink) { - if (data.containsAny(normalized)) { - sink.add(data); - } - }); + return StreamTransformer.fromBind( + (source) => source.where((data) => data.containsAny(normalized))); } } @@ -77,21 +73,22 @@ class UpdateNotification { /// Behaviour: /// If there was no event in "timeout", and one comes in, it is pushed immediately. /// Otherwise, we wait until the timeout is over. -Stream _throttleStream(Stream input, Duration timeout, +Stream _throttleStream(Stream input, Duration timeout, {bool throttleFirst = false, T Function(T, T)? add, T? addOne}) async* { var nextPing = Completer(); + var done = false; T? lastData; var listener = input.listen((data) { - if (lastData is T && add != null) { - lastData = add(lastData as T, data); + if (lastData != null && add != null) { + lastData = add(lastData!, data); } else { lastData = data; } if (!nextPing.isCompleted) { nextPing.complete(); } - }); + }, onDone: () => done = true); try { if (addOne != null) { @@ -100,7 +97,7 @@ Stream _throttleStream(Stream input, Duration timeout, if (throttleFirst) { await Future.delayed(timeout); } - while (true) { + while (!done) { // If a value is available now, we'll use it immediately. // If not, this waits for it. await nextPing.future; @@ -114,6 +111,10 @@ Stream _throttleStream(Stream input, Duration timeout, await Future.delayed(timeout); } } finally { - listener.cancel(); + if (lastData case final data?) { + yield data; + } + + await listener.cancel(); } } diff --git a/packages/sqlite_async/pubspec.yaml b/packages/sqlite_async/pubspec.yaml index d333166..8aac3d6 100644 --- a/packages/sqlite_async/pubspec.yaml +++ b/packages/sqlite_async/pubspec.yaml @@ -34,6 +34,7 @@ dev_dependencies: stream_channel: ^2.1.2 path: ^1.9.0 test_descriptor: ^2.0.2 + fake_async: ^1.3.3 platforms: android: diff --git a/packages/sqlite_async/test/update_notification_test.dart b/packages/sqlite_async/test/update_notification_test.dart new file mode 100644 index 0000000..1410862 --- /dev/null +++ b/packages/sqlite_async/test/update_notification_test.dart @@ -0,0 +1,134 @@ +import 'dart:async'; + +import 'package:fake_async/fake_async.dart'; +import 'package:sqlite_async/src/update_notification.dart'; +import 'package:test/test.dart'; + +void main() { + group('Update notifications', () { + const timeout = Duration(seconds: 10); + const halfTimeout = Duration(seconds: 5); + + group('throttle', () { + test('can add initial', () { + fakeAsync((control) { + final source = StreamController(sync: true); + final events = []; + + UpdateNotification.throttleStream(source.stream, timeout, + addOne: UpdateNotification({'a'})).listen(events.add); + + control.flushMicrotasks(); + expect(events, hasLength(1)); + control.elapse(halfTimeout); + + source.add(UpdateNotification({'b'})); + expect(events, hasLength(1)); // Still a delay from the initial one + + control.elapse(halfTimeout); + expect(events, hasLength(2)); + }); + }); + + test('sends events after initial throttle', () { + fakeAsync((control) { + final source = StreamController(sync: true); + final events = []; + + UpdateNotification.throttleStream(source.stream, timeout) + .listen(events.add); + + source.add(UpdateNotification({'a'})); + control.elapse(halfTimeout); + expect(events, isEmpty); + + control.elapse(halfTimeout); + expect(events, hasLength(1)); + }); + }); + + test('merges events', () { + fakeAsync((control) { + final source = StreamController(sync: true); + final events = []; + + UpdateNotification.throttleStream(source.stream, timeout) + .listen(events.add); + + source.add(UpdateNotification({'a'})); + control.elapse(halfTimeout); + expect(events, isEmpty); + + source.add(UpdateNotification({'b'})); + control.elapse(halfTimeout); + expect(events, [ + UpdateNotification({'a', 'b'}) + ]); + }); + }); + + test('forwards cancellations', () { + fakeAsync((control) { + var cancelled = false; + final source = StreamController(sync: true) + ..onCancel = () => cancelled = true; + + final sub = UpdateNotification.throttleStream(source.stream, timeout) + .listen((_) => fail('unexpected event'), + onDone: () => fail('unexpected done')); + + source.add(UpdateNotification({'a'})); + control.elapse(halfTimeout); + + sub.cancel(); + control.flushTimers(); + + expect(cancelled, isTrue); + expect(control.pendingTimers, isEmpty); + }); + }); + + test('closes when source closes', () { + fakeAsync((control) { + final source = StreamController(sync: true) + ..onCancel = () => Future.value(); + final events = []; + var done = false; + + UpdateNotification.throttleStream(source.stream, timeout) + .listen(events.add, onDone: () => done = true); + + source + // These two are combined due to throttleFirst + ..add(UpdateNotification({'a'})) + ..add(UpdateNotification({'b'})) + ..close(); + + control.flushTimers(); + expect(events, [ + UpdateNotification({'a', 'b'}) + ]); + expect(done, isTrue); + expect(control.pendingTimers, isEmpty); + }); + }); + }); + + test('filter tables', () async { + final source = StreamController(sync: true); + final events = []; + final subscription = UpdateNotification.filterTablesTransformer(['a']) + .bind(source.stream) + .listen(events.add); + + source.add(UpdateNotification({'a', 'b'})); + expect(events, hasLength(1)); + + source.add(UpdateNotification({'b'})); + expect(events, hasLength(1)); + + await subscription.cancel(); + expect(source.hasListener, isFalse); + }); + }); +} From 3a0b54a0ae897900634e6f00b7585a17336a8422 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Wed, 5 Feb 2025 13:16:06 +0100 Subject: [PATCH 48/90] Fix closing after delay --- .../lib/src/update_notification.dart | 10 +++++++++- .../test/update_notification_test.dart | 20 +++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/packages/sqlite_async/lib/src/update_notification.dart b/packages/sqlite_async/lib/src/update_notification.dart index 7832e32..0c8f2c6 100644 --- a/packages/sqlite_async/lib/src/update_notification.dart +++ b/packages/sqlite_async/lib/src/update_notification.dart @@ -88,7 +88,13 @@ Stream _throttleStream(Stream input, Duration timeout, if (!nextPing.isCompleted) { nextPing.complete(); } - }, onDone: () => done = true); + }, onDone: () { + if (!nextPing.isCompleted) { + nextPing.complete(); + } + + done = true; + }); try { if (addOne != null) { @@ -101,6 +107,8 @@ Stream _throttleStream(Stream input, Duration timeout, // If a value is available now, we'll use it immediately. // If not, this waits for it. await nextPing.future; + if (done) break; + // Capture any new values coming in while we wait. nextPing = Completer(); T data = lastData as T; diff --git a/packages/sqlite_async/test/update_notification_test.dart b/packages/sqlite_async/test/update_notification_test.dart index 1410862..0a00ccb 100644 --- a/packages/sqlite_async/test/update_notification_test.dart +++ b/packages/sqlite_async/test/update_notification_test.dart @@ -112,6 +112,26 @@ void main() { expect(control.pendingTimers, isEmpty); }); }); + + test('closes when source closes after delay', () { + fakeAsync((control) { + final source = StreamController(sync: true) + ..onCancel = () => Future.value(); + final events = []; + var done = false; + + UpdateNotification.throttleStream(source.stream, timeout) + .listen(events.add, onDone: () => done = true); + + control.elapse(const Duration(hours: 1)); + source.close(); + + control.flushTimers(); + expect(events, isEmpty); + expect(done, isTrue); + expect(control.pendingTimers, isEmpty); + }); + }); }); test('filter tables', () async { From 8e15de59a8b003655980c71505e96daad18c3347 Mon Sep 17 00:00:00 2001 From: Jorge Sardina Date: Thu, 24 Apr 2025 17:37:48 +0200 Subject: [PATCH 49/90] Fix write detection when using UPDATE/INSERT/DELETE with RETURNING in raw queries (#89) --- packages/drift_sqlite_async/CHANGELOG.md | 4 +++ .../drift_sqlite_async/lib/src/executor.dart | 3 ++- packages/drift_sqlite_async/pubspec.yaml | 2 +- .../drift_sqlite_async/test/basic_test.dart | 25 +++++++++++++++++++ 4 files changed, 32 insertions(+), 2 deletions(-) diff --git a/packages/drift_sqlite_async/CHANGELOG.md b/packages/drift_sqlite_async/CHANGELOG.md index d39db6f..b101ac4 100644 --- a/packages/drift_sqlite_async/CHANGELOG.md +++ b/packages/drift_sqlite_async/CHANGELOG.md @@ -1,3 +1,7 @@ +## 0.2.2 + +- Fix write detection when using UPDATE/INSERT/DELETE with RETURNING in raw queries. + ## 0.2.1 - Fix lints. diff --git a/packages/drift_sqlite_async/lib/src/executor.dart b/packages/drift_sqlite_async/lib/src/executor.dart index dbd4c96..41886f3 100644 --- a/packages/drift_sqlite_async/lib/src/executor.dart +++ b/packages/drift_sqlite_async/lib/src/executor.dart @@ -8,7 +8,8 @@ import 'package:sqlite_async/sqlite_async.dart'; // Ends with " RETURNING *", or starts with insert/update/delete. // Drift-generated queries will always have the RETURNING *. // The INSERT/UPDATE/DELETE check is for custom queries, and is not exhaustive. -final _returningCheck = RegExp(r'( RETURNING \*;?$)|(^(INSERT|UPDATE|DELETE))', +final _returningCheck = RegExp( + r'( RETURNING \*;?\s*$)|(^\s*(INSERT|UPDATE|DELETE))', caseSensitive: false); class _SqliteAsyncDelegate extends _SqliteAsyncQueryDelegate diff --git a/packages/drift_sqlite_async/pubspec.yaml b/packages/drift_sqlite_async/pubspec.yaml index 98eb766..62a64da 100644 --- a/packages/drift_sqlite_async/pubspec.yaml +++ b/packages/drift_sqlite_async/pubspec.yaml @@ -1,5 +1,5 @@ name: drift_sqlite_async -version: 0.2.1 +version: 0.2.2 homepage: https://github.com/powersync-ja/sqlite_async.dart repository: https://github.com/powersync-ja/sqlite_async.dart description: Use Drift with a sqlite_async database, allowing both to be used in the same application. diff --git a/packages/drift_sqlite_async/test/basic_test.dart b/packages/drift_sqlite_async/test/basic_test.dart index 371c149..339cc04 100644 --- a/packages/drift_sqlite_async/test/basic_test.dart +++ b/packages/drift_sqlite_async/test/basic_test.dart @@ -6,6 +6,7 @@ import 'dart:async'; import 'package:drift/drift.dart'; import 'package:drift_sqlite_async/drift_sqlite_async.dart'; +import 'package:sqlite3/common.dart'; import 'package:sqlite_async/sqlite_async.dart'; import 'package:test/test.dart'; @@ -219,5 +220,29 @@ void main() { .data, equals({'count': 1})); }); + + test('cannot update database with read', () async { + await expectLater(() => dbu.customSelect(''' +-- trick to circumvent regex detecting writes +INSERT INTO test_data(description) VALUES('test data'); +''').get(), throwsA(isA())); + }); + + test('allows spaces after returning', () async { + // This tests that the statement is forwarded to the write connection + // despite using customSelect(). If it wasn't, we'd get an error about + // the database being read-only. + final row = await dbu.customSelect( + 'INSERT INTO test_data(description) VALUES(?) RETURNING * ', + variables: [Variable('Test Data')]).getSingle(); + expect(row.data['description'], equals('Test Data')); + }); + + test('allows spaces before insert', () async { + final row = await dbu.customSelect( + ' INSERT INTO test_data(description) VALUES(?) ', + variables: [Variable('Test Data')]).get(); + expect(row, isEmpty); + }); }); } From 6d85217e30c61c3e51372b2304aec6ae1f1660b6 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Mon, 28 Apr 2025 18:08:56 +0200 Subject: [PATCH 50/90] Add test ensuring delete returns affected rows (#90) Add test ensuring delete returns changed rows --- packages/drift_sqlite_async/test/db_test.dart | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/packages/drift_sqlite_async/test/db_test.dart b/packages/drift_sqlite_async/test/db_test.dart index 39a5224..bda09df 100644 --- a/packages/drift_sqlite_async/test/db_test.dart +++ b/packages/drift_sqlite_async/test/db_test.dart @@ -106,5 +106,16 @@ void main() { '[]' ])); }); + + test('delete returns affected rows', () async { + for (var i = 0; i < 10; i++) { + await dbu + .into(dbu.todoItems) + .insert(TodoItemsCompanion.insert(description: 'desc $i')); + } + + final deleted = await dbu.delete(dbu.todoItems).go(); + expect(deleted, 10); + }); }); } From bb65cfad1610eafb44dd48293f644454e81b90de Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Thu, 22 May 2025 10:29:07 +0200 Subject: [PATCH 51/90] Allow profiling queries (#91) * feat: Allow profiling queries * Use Homebrew SQLite if available * More web improvements * chore(release): publish packages - sqlite_async@0.11.5 - drift_sqlite_async@0.2.2+1 * Fix passing task to write context * Enable profiling by default * Add common prefix * Include prefix on web * Disable profiling on workers Browsers don't surface this information either way * Remove pubspec changes to drift_sqlite_async --- packages/sqlite_async/CHANGELOG.md | 4 + .../connection/sync_sqlite_connection.dart | 73 +++++-- .../native_sqlite_connection_impl.dart | 196 +++++++++++------- .../lib/src/sqlite_connection.dart | 8 +- .../sqlite_async/lib/src/sqlite_options.dart | 34 +-- .../sqlite_async/lib/src/utils/profiler.dart | 64 ++++++ .../sqlite_async/lib/src/web/database.dart | 69 ++++-- .../lib/src/web/web_sqlite_open_factory.dart | 3 +- packages/sqlite_async/lib/web.dart | 1 + packages/sqlite_async/pubspec.yaml | 2 +- .../test/utils/native_test_utils.dart | 10 + 11 files changed, 337 insertions(+), 127 deletions(-) create mode 100644 packages/sqlite_async/lib/src/utils/profiler.dart diff --git a/packages/sqlite_async/CHANGELOG.md b/packages/sqlite_async/CHANGELOG.md index 0038ad2..42a5531 100644 --- a/packages/sqlite_async/CHANGELOG.md +++ b/packages/sqlite_async/CHANGELOG.md @@ -1,3 +1,7 @@ +## 0.11.5 + + - Allow profiling queries. + ## 0.11.4 - Add `SqliteConnection.synchronousWrapper` and `SqliteDatabase.singleConnection`. diff --git a/packages/sqlite_async/lib/src/common/connection/sync_sqlite_connection.dart b/packages/sqlite_async/lib/src/common/connection/sync_sqlite_connection.dart index f29520f..d84bdaa 100644 --- a/packages/sqlite_async/lib/src/common/connection/sync_sqlite_connection.dart +++ b/packages/sqlite_async/lib/src/common/connection/sync_sqlite_connection.dart @@ -1,8 +1,12 @@ +import 'dart:developer'; + import 'package:sqlite3/common.dart'; import 'package:sqlite_async/src/common/mutex.dart'; import 'package:sqlite_async/src/sqlite_connection.dart'; +import 'package:sqlite_async/src/sqlite_options.dart'; import 'package:sqlite_async/src/sqlite_queries.dart'; import 'package:sqlite_async/src/update_notification.dart'; +import 'package:sqlite_async/src/utils/profiler.dart'; /// A simple "synchronous" connection which provides the async SqliteConnection /// implementation using a synchronous SQLite connection @@ -14,7 +18,15 @@ class SyncSqliteConnection extends SqliteConnection with SqliteQueries { bool _closed = false; - SyncSqliteConnection(this.db, Mutex m) { + /// Whether queries should be added to the `dart:developer` timeline. + /// + /// This is enabled by default outside of release builds, see + /// [SqliteOptions.profileQueries] for details. + final bool profileQueries; + + SyncSqliteConnection(this.db, Mutex m, {bool? profileQueries}) + : profileQueries = + profileQueries ?? const SqliteOptions().profileQueries { mutex = m.open(); updates = db.updates.map( (event) { @@ -26,15 +38,31 @@ class SyncSqliteConnection extends SqliteConnection with SqliteQueries { @override Future readLock(Future Function(SqliteReadContext tx) callback, {Duration? lockTimeout, String? debugContext}) { - return mutex.lock(() => callback(SyncReadContext(db)), - timeout: lockTimeout); + final task = profileQueries ? TimelineTask() : null; + task?.start('${profilerPrefix}mutex_lock'); + + return mutex.lock( + () { + task?.finish(); + return callback(SyncReadContext(db, parent: task)); + }, + timeout: lockTimeout, + ); } @override Future writeLock(Future Function(SqliteWriteContext tx) callback, {Duration? lockTimeout, String? debugContext}) { - return mutex.lock(() => callback(SyncWriteContext(db)), - timeout: lockTimeout); + final task = profileQueries ? TimelineTask() : null; + task?.start('${profilerPrefix}mutex_lock'); + + return mutex.lock( + () { + task?.finish(); + return callback(SyncWriteContext(db, parent: task)); + }, + timeout: lockTimeout, + ); } @override @@ -53,9 +81,12 @@ class SyncSqliteConnection extends SqliteConnection with SqliteQueries { } class SyncReadContext implements SqliteReadContext { + final TimelineTask? task; + CommonDatabase db; - SyncReadContext(this.db); + SyncReadContext(this.db, {TimelineTask? parent}) + : task = TimelineTask(parent: parent); @override Future computeWithDatabase( @@ -65,13 +96,23 @@ class SyncReadContext implements SqliteReadContext { @override Future get(String sql, [List parameters = const []]) async { - return db.select(sql, parameters).first; + return task.timeSync( + 'get', + () => db.select(sql, parameters).first, + sql: sql, + parameters: parameters, + ); } @override Future getAll(String sql, [List parameters = const []]) async { - return db.select(sql, parameters); + return task.timeSync( + 'getAll', + () => db.select(sql, parameters), + sql: sql, + parameters: parameters, + ); } @override @@ -91,26 +132,32 @@ class SyncReadContext implements SqliteReadContext { } class SyncWriteContext extends SyncReadContext implements SqliteWriteContext { - SyncWriteContext(super.db); + SyncWriteContext(super.db, {super.parent}); @override Future execute(String sql, [List parameters = const []]) async { - return db.select(sql, parameters); + return task.timeSync( + 'execute', + () => db.select(sql, parameters), + sql: sql, + parameters: parameters, + ); } @override Future executeBatch( String sql, List> parameterSets) async { - return computeWithDatabase((db) async { + task.timeSync('executeBatch', () { final statement = db.prepare(sql, checkNoTail: true); try { for (var parameters in parameterSets) { - statement.execute(parameters); + task.timeSync('iteration', () => statement.execute(parameters), + parameters: parameters); } } finally { statement.dispose(); } - }); + }, sql: sql); } } diff --git a/packages/sqlite_async/lib/src/native/database/native_sqlite_connection_impl.dart b/packages/sqlite_async/lib/src/native/database/native_sqlite_connection_impl.dart index 7df4ac8..ae3c376 100644 --- a/packages/sqlite_async/lib/src/native/database/native_sqlite_connection_impl.dart +++ b/packages/sqlite_async/lib/src/native/database/native_sqlite_connection_impl.dart @@ -1,4 +1,5 @@ import 'dart:async'; +import 'dart:developer'; import 'dart:isolate'; import 'package:sqlite3/sqlite3.dart' as sqlite; @@ -10,6 +11,7 @@ import 'package:sqlite_async/src/native/native_isolate_mutex.dart'; import 'package:sqlite_async/src/sqlite_connection.dart'; import 'package:sqlite_async/src/sqlite_queries.dart'; import 'package:sqlite_async/src/update_notification.dart'; +import 'package:sqlite_async/src/utils/profiler.dart'; import 'package:sqlite_async/src/utils/shared_utils.dart'; import 'upstream_updates.dart'; @@ -33,15 +35,18 @@ class SqliteConnectionImpl final String? debugName; final bool readOnly; - SqliteConnectionImpl( - {required openFactory, - required Mutex mutex, - SerializedPortClient? upstreamPort, - Stream? updates, - this.debugName, - this.readOnly = false, - bool primary = false}) - : _writeMutex = mutex { + final bool profileQueries; + + SqliteConnectionImpl({ + required AbstractDefaultSqliteOpenFactory openFactory, + required Mutex mutex, + SerializedPortClient? upstreamPort, + Stream? updates, + this.debugName, + this.readOnly = false, + bool primary = false, + }) : _writeMutex = mutex, + profileQueries = openFactory.sqliteOptions.profileQueries { isInitialized = _isolateClient.ready; this.upstreamPort = upstreamPort ?? listenForEvents(); // Accept an incoming stream of updates, or expose one if not given. @@ -58,6 +63,11 @@ class SqliteConnectionImpl return _isolateClient.closed; } + _TransactionContext _context() { + return _TransactionContext( + _isolateClient, profileQueries ? TimelineTask() : null); + } + @override Future getAutoCommit() async { if (closed) { @@ -65,7 +75,7 @@ class SqliteConnectionImpl } // We use a _TransactionContext without a lock here. // It is safe to call this in the middle of another transaction. - final ctx = _TransactionContext(_isolateClient); + final ctx = _context(); try { return await ctx.getAutoCommit(); } finally { @@ -120,7 +130,7 @@ class SqliteConnectionImpl // Private lock to synchronize this with other statements on the same connection, // to ensure that transactions aren't interleaved. return _connectionMutex.lock(() async { - final ctx = _TransactionContext(_isolateClient); + final ctx = _context(); try { return await callback(ctx); } finally { @@ -143,7 +153,7 @@ class SqliteConnectionImpl } // DB lock so that only one write happens at a time return await _writeMutex.lock(() async { - final ctx = _TransactionContext(_isolateClient); + final ctx = _context(); try { return await callback(ctx); } finally { @@ -167,7 +177,9 @@ class _TransactionContext implements SqliteWriteContext { bool _closed = false; final int ctxId = _nextCtxId++; - _TransactionContext(this._sendPort); + final TimelineTask? task; + + _TransactionContext(this._sendPort, this.task); @override bool get closed { @@ -187,8 +199,13 @@ class _TransactionContext implements SqliteWriteContext { throw sqlite.SqliteException(0, 'Transaction closed', null, sql); } try { - var future = _sendPort.post( - _SqliteIsolateStatement(ctxId, sql, parameters, readOnly: false)); + var future = _sendPort.post(_SqliteIsolateStatement( + ctxId, + sql, + parameters, + readOnly: false, + timelineTask: task?.pass(), + )); return await future; } on sqlite.SqliteException catch (e) { @@ -314,74 +331,101 @@ Future _sqliteConnectionIsolateInner(_SqliteConnectionParams params, Timer(const Duration(milliseconds: 1), maybeFireUpdates); }); - server.open((data) async { - if (data is _SqliteIsolateClose) { - // This is a transaction close message + ResultSet runStatement(_SqliteIsolateStatement data) { + if (data.sql == 'BEGIN' || data.sql == 'BEGIN IMMEDIATE') { if (txId != null) { - if (!db.autocommit) { - db.execute('ROLLBACK'); - } - txId = null; - txError = null; - throw sqlite.SqliteException( - 0, 'Transaction must be closed within the read or write lock'); + // This will error on db.select } - // We would likely have received updates by this point - fire now. - maybeFireUpdates(); - return null; - } else if (data is _SqliteIsolateStatement) { - if (data.sql == 'BEGIN' || data.sql == 'BEGIN IMMEDIATE') { - if (txId != null) { - // This will error on db.select + txId = data.ctxId; + } else if (txId != null && txId != data.ctxId) { + // Locks should prevent this from happening + throw sqlite.SqliteException( + 0, 'Mixed transactions: $txId and ${data.ctxId}'); + } else if (data.sql == 'ROLLBACK') { + // This is the only valid way to clear an error + txError = null; + txId = null; + } else if (txError != null) { + // Any statement (including COMMIT) after the first error will also error, until the + // transaction is aborted. + throw txError!; + } else if (data.sql == 'COMMIT' || data.sql == 'END TRANSACTION') { + txId = null; + } + try { + final result = db.select(data.sql, mapParameters(data.args)); + return result; + } catch (err) { + if (txId != null) { + if (db.autocommit) { + // Transaction rolled back + txError = sqlite.SqliteException(0, + 'Transaction rolled back by earlier statement: ${err.toString()}'); + } else { + // Recoverable error } - txId = data.ctxId; - } else if (txId != null && txId != data.ctxId) { - // Locks should prevent this from happening - throw sqlite.SqliteException( - 0, 'Mixed transactions: $txId and ${data.ctxId}'); - } else if (data.sql == 'ROLLBACK') { - // This is the only valid way to clear an error - txError = null; - txId = null; - } else if (txError != null) { - // Any statement (including COMMIT) after the first error will also error, until the - // transaction is aborted. - throw txError!; - } else if (data.sql == 'COMMIT' || data.sql == 'END TRANSACTION') { - txId = null; } - try { - final result = db.select(data.sql, mapParameters(data.args)); - return result; - } catch (err) { + rethrow; + } + } + + Future handle(_RemoteIsolateRequest data, TimelineTask? task) async { + switch (data) { + case _SqliteIsolateClose(): + // This is a transaction close message if (txId != null) { - if (db.autocommit) { - // Transaction rolled back - txError = sqlite.SqliteException(0, - 'Transaction rolled back by earlier statement: ${err.toString()}'); - } else { - // Recoverable error + if (!db.autocommit) { + db.execute('ROLLBACK'); } + txId = null; + txError = null; + throw sqlite.SqliteException( + 0, 'Transaction must be closed within the read or write lock'); } - rethrow; - } - } else if (data is _SqliteIsolateClosure) { - try { - return await data.cb(db); - } finally { + // We would likely have received updates by this point - fire now. maybeFireUpdates(); - } - } else if (data is _SqliteIsolateConnectionClose) { - db.dispose(); - return null; - } else { + return null; + case _SqliteIsolateStatement(): + return task.timeSync( + 'execute_remote', + () => runStatement(data), + sql: data.sql, + parameters: data.args, + ); + case _SqliteIsolateClosure(): + try { + return await data.cb(db); + } finally { + maybeFireUpdates(); + } + case _SqliteIsolateConnectionClose(): + db.dispose(); + return null; + } + } + + server.open((data) async { + if (data is! _RemoteIsolateRequest) { throw ArgumentError('Unknown data type $data'); } + + final task = switch (data.timelineTask) { + null => null, + final id => TimelineTask.withTaskId(id), + }; + + return await handle(data, task); }); commandPort.listen((data) async {}); } +sealed class _RemoteIsolateRequest { + final int? timelineTask; + + const _RemoteIsolateRequest({required this.timelineTask}); +} + class _SqliteConnectionParams { final RequestPortServer portServer; final bool readOnly; @@ -398,28 +442,28 @@ class _SqliteConnectionParams { required this.primary}); } -class _SqliteIsolateStatement { +class _SqliteIsolateStatement extends _RemoteIsolateRequest { final int ctxId; final String sql; final List args; final bool readOnly; _SqliteIsolateStatement(this.ctxId, this.sql, this.args, - {this.readOnly = false}); + {this.readOnly = false, super.timelineTask}); } -class _SqliteIsolateClosure { +class _SqliteIsolateClosure extends _RemoteIsolateRequest { final TxCallback cb; - _SqliteIsolateClosure(this.cb); + _SqliteIsolateClosure(this.cb, {super.timelineTask}); } -class _SqliteIsolateClose { +class _SqliteIsolateClose extends _RemoteIsolateRequest { final int ctxId; - const _SqliteIsolateClose(this.ctxId); + const _SqliteIsolateClose(this.ctxId, {super.timelineTask}); } -class _SqliteIsolateConnectionClose { - const _SqliteIsolateConnectionClose(); +class _SqliteIsolateConnectionClose extends _RemoteIsolateRequest { + const _SqliteIsolateConnectionClose({super.timelineTask}); } diff --git a/packages/sqlite_async/lib/src/sqlite_connection.dart b/packages/sqlite_async/lib/src/sqlite_connection.dart index 15f4f6a..bd2292b 100644 --- a/packages/sqlite_async/lib/src/sqlite_connection.dart +++ b/packages/sqlite_async/lib/src/sqlite_connection.dart @@ -93,9 +93,13 @@ abstract class SqliteConnection extends SqliteWriteContext { /// may be easier to wrap a [raw] databases (like unit tests), this method /// may be used as an escape hatch for the asynchronous wrappers provided by /// this package. + /// + /// When [profileQueries] is enabled (it's enabled by default outside of + /// release builds, queries are posted to the `dart:developer` timeline). factory SqliteConnection.synchronousWrapper(CommonDatabase raw, - {Mutex? mutex}) { - return SyncSqliteConnection(raw, mutex ?? Mutex()); + {Mutex? mutex, bool? profileQueries}) { + return SyncSqliteConnection(raw, mutex ?? Mutex(), + profileQueries: profileQueries); } /// Reports table change update notifications diff --git a/packages/sqlite_async/lib/src/sqlite_options.dart b/packages/sqlite_async/lib/src/sqlite_options.dart index 3767213..0489f6b 100644 --- a/packages/sqlite_async/lib/src/sqlite_options.dart +++ b/packages/sqlite_async/lib/src/sqlite_options.dart @@ -30,19 +30,27 @@ class SqliteOptions { /// Set to null or [Duration.zero] to fail immediately when the database is locked. final Duration? lockTimeout; - const SqliteOptions.defaults() - : journalMode = SqliteJournalMode.wal, - journalSizeLimit = 6 * 1024 * 1024, // 1.5x the default checkpoint size - synchronous = SqliteSynchronous.normal, - webSqliteOptions = const WebSqliteOptions.defaults(), - lockTimeout = const Duration(seconds: 30); - - const SqliteOptions( - {this.journalMode = SqliteJournalMode.wal, - this.journalSizeLimit = 6 * 1024 * 1024, - this.synchronous = SqliteSynchronous.normal, - this.webSqliteOptions = const WebSqliteOptions.defaults(), - this.lockTimeout = const Duration(seconds: 30)}); + /// Whether queries should be added to the `dart:developer` timeline. + /// + /// By default, this is enabled if the `dart.vm.product` compile-time variable + /// is not set to `true`. For Flutter apps, this means that [profileQueries] + /// is enabled by default in debug and profile mode. + final bool profileQueries; + + const factory SqliteOptions.defaults() = SqliteOptions; + + const SqliteOptions({ + this.journalMode = SqliteJournalMode.wal, + this.journalSizeLimit = 6 * 1024 * 1024, + this.synchronous = SqliteSynchronous.normal, + this.webSqliteOptions = const WebSqliteOptions.defaults(), + this.lockTimeout = const Duration(seconds: 30), + this.profileQueries = _profileQueriesByDefault, + }); + + // https://api.flutter.dev/flutter/foundation/kReleaseMode-constant.html + static const _profileQueriesByDefault = + !bool.fromEnvironment('dart.vm.product'); } /// SQLite journal mode. Set on the primary connection. diff --git a/packages/sqlite_async/lib/src/utils/profiler.dart b/packages/sqlite_async/lib/src/utils/profiler.dart new file mode 100644 index 0000000..bffbf39 --- /dev/null +++ b/packages/sqlite_async/lib/src/utils/profiler.dart @@ -0,0 +1,64 @@ +import 'dart:developer'; + +extension TimeSync on TimelineTask? { + T timeSync(String name, TimelineSyncFunction function, + {String? sql, List? parameters}) { + final currentTask = this; + if (currentTask == null) { + return function(); + } + + final (resolvedName, args) = + profilerNameAndArgs(name, sql: sql, parameters: parameters); + currentTask.start(resolvedName, arguments: args); + + try { + return function(); + } finally { + currentTask.finish(); + } + } + + Future timeAsync(String name, TimelineSyncFunction> function, + {String? sql, List? parameters}) { + final currentTask = this; + if (currentTask == null) { + return function(); + } + + final (resolvedName, args) = + profilerNameAndArgs(name, sql: sql, parameters: parameters); + currentTask.start(resolvedName, arguments: args); + + return Future.sync(function).whenComplete(() { + currentTask.finish(); + }); + } +} + +(String, Map) profilerNameAndArgs(String name, + {String? sql, List? parameters}) { + // On native platforms, we want static names for tasks because every + // unique key here shows up in a separate line in Perfetto: https://github.com/dart-lang/sdk/issues/56274 + // On the web however, the names are embedded in the timeline slices and + // it's convenient to include the SQL there. + const isWeb = bool.fromEnvironment('dart.library.js_interop'); + var resolvedName = '$profilerPrefix$name'; + if (isWeb && sql != null) { + resolvedName = '$resolvedName $sql'; + } + + return ( + resolvedName, + { + if (sql != null) 'sql': sql, + if (parameters != null) + 'parameters': [ + for (final parameter in parameters) + if (parameter is List) '' else parameter + ], + } + ); +} + +const profilerPrefix = 'sqlite_async:'; diff --git a/packages/sqlite_async/lib/src/web/database.dart b/packages/sqlite_async/lib/src/web/database.dart index 04da846..6ba7e6b 100644 --- a/packages/sqlite_async/lib/src/web/database.dart +++ b/packages/sqlite_async/lib/src/web/database.dart @@ -1,4 +1,5 @@ import 'dart:async'; +import 'dart:developer'; import 'dart:js_interop'; import 'dart:js_interop_unsafe'; @@ -6,6 +7,7 @@ import 'package:sqlite3/common.dart'; import 'package:sqlite3_web/sqlite3_web.dart'; import 'package:sqlite3_web/protocol_utils.dart' as proto; import 'package:sqlite_async/sqlite_async.dart'; +import 'package:sqlite_async/src/utils/profiler.dart'; import 'package:sqlite_async/src/utils/shared_utils.dart'; import 'package:sqlite_async/src/web/database/broadcast_updates.dart'; import 'package:sqlite_async/web.dart'; @@ -17,6 +19,7 @@ class WebDatabase implements SqliteDatabase, WebSqliteConnection { final Database _database; final Mutex? _mutex; + final bool profileQueries; /// For persistent databases that aren't backed by a shared worker, we use /// web broadcast channels to forward local update events to other tabs. @@ -25,7 +28,12 @@ class WebDatabase @override bool closed = false; - WebDatabase(this._database, this._mutex, {this.broadcastUpdates}); + WebDatabase( + this._database, + this._mutex, { + required this.profileQueries, + this.broadcastUpdates, + }); @override Future close() async { @@ -175,7 +183,10 @@ class _SharedContext implements SqliteReadContext { final WebDatabase _database; bool _contextClosed = false; - _SharedContext(this._database); + final TimelineTask? _task; + + _SharedContext(this._database) + : _task = _database.profileQueries ? TimelineTask() : null; @override bool get closed => _contextClosed || _database.closed; @@ -196,8 +207,15 @@ class _SharedContext implements SqliteReadContext { @override Future getAll(String sql, [List parameters = const []]) async { - return await wrapSqliteException( - () => _database._database.select(sql, parameters)); + return _task.timeAsync( + 'getAll', + sql: sql, + parameters: parameters, + () async { + return await wrapSqliteException( + () => _database._database.select(sql, parameters)); + }, + ); } @override @@ -221,35 +239,37 @@ class _ExclusiveContext extends _SharedContext implements SqliteWriteContext { _ExclusiveContext(super.database); @override - Future execute(String sql, - [List parameters = const []]) async { - return wrapSqliteException( - () => _database._database.select(sql, parameters)); + Future execute(String sql, [List parameters = const []]) { + return _task.timeAsync('execute', sql: sql, parameters: parameters, () { + return wrapSqliteException( + () => _database._database.select(sql, parameters)); + }); } @override - Future executeBatch( - String sql, List> parameterSets) async { - return wrapSqliteException(() async { - for (final set in parameterSets) { - // use execute instead of select to avoid transferring rows from the - // worker to this context. - await _database._database.execute(sql, set); - } + Future executeBatch(String sql, List> parameterSets) { + return _task.timeAsync('executeBatch', sql: sql, () { + return wrapSqliteException(() async { + for (final set in parameterSets) { + // use execute instead of select to avoid transferring rows from the + // worker to this context. + await _database._database.execute(sql, set); + } + }); }); } } class _ExclusiveTransactionContext extends _ExclusiveContext { SqliteWriteContext baseContext; + _ExclusiveTransactionContext(super.database, this.baseContext); @override bool get closed => baseContext.closed; - @override - Future execute(String sql, - [List parameters = const []]) async { + Future _executeInternal( + String sql, List parameters) async { // Operations inside transactions are executed with custom requests // in order to verify that the connection does not have autocommit enabled. // The worker will check if autocommit = true before executing the SQL. @@ -293,15 +313,22 @@ class _ExclusiveTransactionContext extends _ExclusiveContext { }); } + @override + Future execute(String sql, + [List parameters = const []]) async { + return _task.timeAsync('execute', sql: sql, parameters: parameters, () { + return _executeInternal(sql, parameters); + }); + } + @override Future executeBatch( String sql, List> parameterSets) async { - return await wrapSqliteException(() async { + return _task.timeAsync('executeBatch', sql: sql, () async { for (final set in parameterSets) { await _database._database.customRequest(CustomDatabaseMessage( CustomDatabaseMessageKind.executeBatchInTransaction, sql, set)); } - return; }); } } diff --git a/packages/sqlite_async/lib/src/web/web_sqlite_open_factory.dart b/packages/sqlite_async/lib/src/web/web_sqlite_open_factory.dart index 5089b95..a724329 100644 --- a/packages/sqlite_async/lib/src/web/web_sqlite_open_factory.dart +++ b/packages/sqlite_async/lib/src/web/web_sqlite_open_factory.dart @@ -76,7 +76,8 @@ class DefaultSqliteOpenFactory } return WebDatabase(connection.database, options.mutex ?? mutex, - broadcastUpdates: updates); + broadcastUpdates: updates, + profileQueries: sqliteOptions.profileQueries); } @override diff --git a/packages/sqlite_async/lib/web.dart b/packages/sqlite_async/lib/web.dart index 8051edb..3a65115 100644 --- a/packages/sqlite_async/lib/web.dart +++ b/packages/sqlite_async/lib/web.dart @@ -94,6 +94,7 @@ abstract class WebSqliteConnection implements SqliteConnection { var lock? => Mutex(identifier: lock), null => null, }, + profileQueries: false, ); return database; } diff --git a/packages/sqlite_async/pubspec.yaml b/packages/sqlite_async/pubspec.yaml index 8aac3d6..5142162 100644 --- a/packages/sqlite_async/pubspec.yaml +++ b/packages/sqlite_async/pubspec.yaml @@ -1,6 +1,6 @@ name: sqlite_async description: High-performance asynchronous interface for SQLite on Dart and Flutter. -version: 0.11.4 +version: 0.11.5 repository: https://github.com/powersync-ja/sqlite_async.dart environment: sdk: ">=3.5.0 <4.0.0" diff --git a/packages/sqlite_async/test/utils/native_test_utils.dart b/packages/sqlite_async/test/utils/native_test_utils.dart index 945529d..83dea18 100644 --- a/packages/sqlite_async/test/utils/native_test_utils.dart +++ b/packages/sqlite_async/test/utils/native_test_utils.dart @@ -26,6 +26,16 @@ class TestSqliteOpenFactory extends TestDefaultSqliteOpenFactory { sqlite_open.open.overrideFor(sqlite_open.OperatingSystem.linux, () { return DynamicLibrary.open(sqlitePath); }); + + sqlite_open.open.overrideFor(sqlite_open.OperatingSystem.macOS, () { + // Prefer using Homebrew's SQLite which allows loading extensions. + const fromHomebrew = '/opt/homebrew/opt/sqlite/lib/libsqlite3.dylib'; + if (File(fromHomebrew).existsSync()) { + return DynamicLibrary.open(fromHomebrew); + } + + return DynamicLibrary.open('libsqlite3.dylib'); + }); } @override From ae7dc0e56be0d6a4372e829beb7d5251df2ab44f Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Thu, 22 May 2025 11:13:04 +0200 Subject: [PATCH 52/90] Fix web database not respecting lock timeouts (#88) --- .../sqlite_async/lib/src/web/database.dart | 4 ++-- .../sqlite_async/lib/src/web/web_mutex.dart | 5 ++--- packages/sqlite_async/test/basic_test.dart | 21 +++++++++++++++++++ 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/packages/sqlite_async/lib/src/web/database.dart b/packages/sqlite_async/lib/src/web/database.dart index 6ba7e6b..cd7e215 100644 --- a/packages/sqlite_async/lib/src/web/database.dart +++ b/packages/sqlite_async/lib/src/web/database.dart @@ -94,7 +94,7 @@ class WebDatabase Future readLock(Future Function(SqliteReadContext tx) callback, {Duration? lockTimeout, String? debugContext}) async { if (_mutex case var mutex?) { - return await mutex.lock(() async { + return await mutex.lock(timeout: lockTimeout, () async { final context = _SharedContext(this); try { return await callback(context); @@ -143,7 +143,7 @@ class WebDatabase Future writeLock(Future Function(SqliteWriteContext tx) callback, {Duration? lockTimeout, String? debugContext, bool? flush}) async { if (_mutex case var mutex?) { - return await mutex.lock(() async { + return await mutex.lock(timeout: lockTimeout, () async { final context = _ExclusiveContext(this); try { return await callback(context); diff --git a/packages/sqlite_async/lib/src/web/web_mutex.dart b/packages/sqlite_async/lib/src/web/web_mutex.dart index 8c2baa5..6972b2f 100644 --- a/packages/sqlite_async/lib/src/web/web_mutex.dart +++ b/packages/sqlite_async/lib/src/web/web_mutex.dart @@ -16,10 +16,9 @@ external Navigator get _navigator; /// Web implementation of [Mutex] class MutexImpl implements Mutex { late final mutex.Mutex fallback; - String? identifier; final String resolvedIdentifier; - MutexImpl({this.identifier}) + MutexImpl({String? identifier}) /// On web a lock name is required for Navigator locks. /// Having exclusive Mutex instances requires a somewhat unique lock name. @@ -40,7 +39,7 @@ class MutexImpl implements Mutex { @override Future lock(Future Function() callback, {Duration? timeout}) { - if ((_navigator as JSObject).hasProperty('locks'.toJS).toDart) { + if (_navigator.has('locks')) { return _webLock(callback, timeout: timeout); } else { return _fallbackLock(callback, timeout: timeout); diff --git a/packages/sqlite_async/test/basic_test.dart b/packages/sqlite_async/test/basic_test.dart index 6ee038a..ed2f708 100644 --- a/packages/sqlite_async/test/basic_test.dart +++ b/packages/sqlite_async/test/basic_test.dart @@ -209,6 +209,27 @@ void main() { expect(await db.get('SELECT description FROM test_data'), {'description': 'test'}); }); + + test('respects lock timeouts', () async { + // Unfortunately this test can't use fakeAsync because it uses actual + // lock APIs on the web. + final db = await testUtils.setupDatabase(path: path); + final lockAcquired = Completer(); + + final completion = db.writeLock((context) async { + lockAcquired.complete(); + await Future.delayed(const Duration(seconds: 1)); + }); + + await lockAcquired.future; + await expectLater( + () => db.writeLock( + lockTimeout: Duration(milliseconds: 200), (_) async => {}), + throwsA(isA()), + ); + + await completion; + }); }); } From 238853b74c54647f37421d5c5ab17a29a4c0d9c0 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Thu, 22 May 2025 12:11:21 +0200 Subject: [PATCH 53/90] Make watch queries easier to cancel (#92) * Refactor watch queries to allow cancelling them * Fix handling pause immediately after emit * Review feedback --- .../sqlite_async/lib/src/sqlite_queries.dart | 30 ++-- .../lib/src/update_notification.dart | 165 ++++++++++++------ .../test/update_notification_test.dart | 55 ++++++ packages/sqlite_async/test/watch_test.dart | 8 +- 4 files changed, 187 insertions(+), 71 deletions(-) diff --git a/packages/sqlite_async/lib/src/sqlite_queries.dart b/packages/sqlite_async/lib/src/sqlite_queries.dart index dc91dd1..80bc568 100644 --- a/packages/sqlite_async/lib/src/sqlite_queries.dart +++ b/packages/sqlite_async/lib/src/sqlite_queries.dart @@ -44,25 +44,23 @@ mixin SqliteQueries implements SqliteWriteContext, SqliteConnection { Stream watch(String sql, {List parameters = const [], Duration throttle = const Duration(milliseconds: 30), - Iterable? triggerOnTables}) async* { + Iterable? triggerOnTables}) { assert(updates != null, 'updates stream must be provided to allow query watching'); - final tables = - triggerOnTables ?? await getSourceTables(this, sql, parameters); - final filteredStream = - updates!.transform(UpdateNotification.filterTablesTransformer(tables)); - final throttledStream = UpdateNotification.throttleStream( - filteredStream, throttle, - addOne: UpdateNotification.empty()); - // FIXME: - // When the subscription is cancelled, this performs a final query on the next - // update. - // The loop only stops once the "yield" is reached. - // Using asyncMap instead of a generator would solve it, but then the body - // here can't be async for getSourceTables(). - await for (var _ in throttledStream) { - yield await getAll(sql, parameters); + Stream watchInner(Iterable trigger) { + return onChange( + trigger, + throttle: throttle, + triggerImmediately: true, + ).asyncMap((_) => getAll(sql, parameters)); + } + + if (triggerOnTables case final knownTrigger?) { + return watchInner(knownTrigger); + } else { + return Stream.fromFuture(getSourceTables(this, sql, parameters)) + .asyncExpand(watchInner); } } diff --git a/packages/sqlite_async/lib/src/update_notification.dart b/packages/sqlite_async/lib/src/update_notification.dart index 0c8f2c6..21f3541 100644 --- a/packages/sqlite_async/lib/src/update_notification.dart +++ b/packages/sqlite_async/lib/src/update_notification.dart @@ -52,10 +52,13 @@ class UpdateNotification { static Stream throttleStream( Stream input, Duration timeout, {UpdateNotification? addOne}) { - return _throttleStream(input, timeout, addOne: addOne, throttleFirst: true, - add: (a, b) { - return a.union(b); - }); + return _throttleStream( + input: input, + timeout: timeout, + throttleFirst: true, + add: (a, b) => a.union(b), + addOne: addOne, + ); } /// Filter an update stream by specific tables. @@ -67,62 +70,120 @@ class UpdateNotification { } } -/// Given a broadcast stream, return a singular throttled stream that is throttled. -/// This immediately starts listening. +/// Throttles an [input] stream to not emit events more often than with a +/// frequency of 1/[timeout]. /// -/// Behaviour: -/// If there was no event in "timeout", and one comes in, it is pushed immediately. -/// Otherwise, we wait until the timeout is over. -Stream _throttleStream(Stream input, Duration timeout, - {bool throttleFirst = false, T Function(T, T)? add, T? addOne}) async* { - var nextPing = Completer(); - var done = false; - T? lastData; - - var listener = input.listen((data) { - if (lastData != null && add != null) { - lastData = add(lastData!, data); - } else { - lastData = data; +/// When an event is received and no timeout window is active, it is forwarded +/// downstream and a timeout window is started. For events received within a +/// timeout window, [add] is called to fold events. Then when the window +/// expires, pending events are emitted. +/// The subscription to the [input] stream is never paused. +/// +/// When the returned stream is paused, an active timeout window is reset and +/// restarts after the stream is resumed. +/// +/// If [addOne] is not null, that event will always be added when the stream is +/// subscribed to. +/// When [throttleFirst] is true, a timeout window begins immediately after +/// listening (so that the first event, apart from [addOne], is emitted no +/// earlier than after [timeout]). +Stream _throttleStream({ + required Stream input, + required Duration timeout, + required bool throttleFirst, + required T Function(T, T) add, + required T? addOne, +}) { + return Stream.multi((listener) { + T? pendingData; + Timer? activeTimeoutWindow; + var needsTimeoutWindowAfterResume = false; + + /// Add pending data, bypassing the active timeout window. + /// + /// This is used to forward error and done events immediately. + bool addPendingEvents() { + if (pendingData case final data?) { + pendingData = null; + listener.addSync(data); + activeTimeoutWindow?.cancel(); + activeTimeoutWindow = null; + return true; + } else { + return false; + } } - if (!nextPing.isCompleted) { - nextPing.complete(); + + late void Function() setTimeout; + + /// Emits [pendingData] if no timeout window is active, and then starts a + /// timeout window if necessary. + void maybeEmit() { + if (activeTimeoutWindow == null && !listener.isPaused) { + final didAdd = addPendingEvents(); + if (didAdd) { + // Schedule a pause after resume if the subscription was paused + // directly in response to receiving the event. Otherwise, begin the + // timeout window immediately. + if (listener.isPaused) { + needsTimeoutWindowAfterResume = true; + } else { + setTimeout(); + } + } + } } - }, onDone: () { - if (!nextPing.isCompleted) { - nextPing.complete(); + + setTimeout = () { + activeTimeoutWindow = Timer(timeout, () { + activeTimeoutWindow = null; + maybeEmit(); + }); + }; + + void onData(T data) { + pendingData = switch (pendingData) { + null => data, + final pending => add(pending, data), + }; + maybeEmit(); } - done = true; - }); + void onError(Object error, StackTrace trace) { + addPendingEvents(); + listener.addErrorSync(error, trace); + } + + void onDone() { + addPendingEvents(); + listener.closeSync(); + } + + final subscription = input.listen(onData, onError: onError, onDone: onDone); + + listener.onPause = () { + needsTimeoutWindowAfterResume = activeTimeoutWindow != null; + activeTimeoutWindow?.cancel(); + activeTimeoutWindow = null; + }; + listener.onResume = () { + if (needsTimeoutWindowAfterResume) { + setTimeout(); + } else { + maybeEmit(); + } + }; + listener.onCancel = () async { + activeTimeoutWindow?.cancel(); + return subscription.cancel(); + }; - try { if (addOne != null) { - yield addOne; + // This must not be sync, we're doing this directly in onListen + listener.add(addOne); } if (throttleFirst) { - await Future.delayed(timeout); - } - while (!done) { - // If a value is available now, we'll use it immediately. - // If not, this waits for it. - await nextPing.future; - if (done) break; - - // Capture any new values coming in while we wait. - nextPing = Completer(); - T data = lastData as T; - // Clear before we yield, so that we capture new changes while yielding - lastData = null; - yield data; - // Wait a minimum of this duration between tasks - await Future.delayed(timeout); + setTimeout(); } - } finally { - if (lastData case final data?) { - yield data; - } - - await listener.cancel(); - } + }); } diff --git a/packages/sqlite_async/test/update_notification_test.dart b/packages/sqlite_async/test/update_notification_test.dart index 0a00ccb..05c94c6 100644 --- a/packages/sqlite_async/test/update_notification_test.dart +++ b/packages/sqlite_async/test/update_notification_test.dart @@ -47,6 +47,61 @@ void main() { }); }); + test('increases delay after pause', () { + fakeAsync((control) { + final source = StreamController(sync: true); + final events = []; + + final sub = UpdateNotification.throttleStream(source.stream, timeout) + .listen(null); + sub.onData((event) { + events.add(event); + sub.pause(); + }); + + source.add(UpdateNotification({'a'})); + control.elapse(timeout); + expect(events, hasLength(1)); + + // Assume the stream stays paused for the timeout window that would + // be created after emitting the notification. + control.elapse(timeout * 2); + source.add(UpdateNotification({'b'})); + control.elapse(timeout * 2); + + // A full timeout needs to pass after resuming before a new item is + // emitted. + sub.resume(); + expect(events, hasLength(1)); + + control.elapse(halfTimeout); + expect(events, hasLength(1)); + control.elapse(halfTimeout); + expect(events, hasLength(2)); + }); + }); + + test('does not introduce artificial delay in pause', () { + fakeAsync((control) { + final source = StreamController(sync: true); + final events = []; + + final sub = UpdateNotification.throttleStream(source.stream, timeout) + .listen(events.add); + + // Await the initial delay + control.elapse(timeout); + + sub.pause(); + source.add(UpdateNotification({'a'})); + // Resuming should not introduce a timeout window because no window + // was active when the stream was paused. + sub.resume(); + control.flushMicrotasks(); + expect(events, hasLength(1)); + }); + }); + test('merges events', () { fakeAsync((control) { final source = StreamController(sync: true); diff --git a/packages/sqlite_async/test/watch_test.dart b/packages/sqlite_async/test/watch_test.dart index 7e74790..dc28add 100644 --- a/packages/sqlite_async/test/watch_test.dart +++ b/packages/sqlite_async/test/watch_test.dart @@ -113,8 +113,10 @@ void main() { lastCount = count; } - // The number of read queries must not be greater than the number of writes overall. - expect(numberOfQueries, lessThanOrEqualTo(results.last.first['count'])); + // The number of read queries must not be greater than the number of + // writes overall, plus one for the initial stream emission. + expect(numberOfQueries, + lessThanOrEqualTo(results.last.first['count'] + 1)); DateTime? lastTime; for (var r in times) { @@ -283,7 +285,7 @@ void main() { }); await Future.delayed(delay); - subscription.cancel(); + await subscription.cancel(); expect( counts, From dd7c810c7ffb1c1d0b15816da0cb6ed90ba47f36 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Thu, 22 May 2025 12:16:26 +0200 Subject: [PATCH 54/90] Prepare release of sqlite_async 0.11.5 --- CHANGELOG.md | 21 +++++++++++++++++++++ packages/sqlite_async/CHANGELOG.md | 5 ++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aba9311..d23d6d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,27 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## 2025-05-22 + +--- + +Packages with breaking changes: + + - There are no breaking changes in this release. + +Packages with other changes: + + - [`sqlite_async` - `v0.11.5`](#sqlite_async---v0115) + +--- + +#### `sqlite_async` - `v0.11.5` + +- Allow profiling queries. Queries are profiled by default in debug and profile builds, the runtime + for queries is added to profiling timelines under the `sqlite_async` tag. +- Fix cancelling `watch()` queries sometimes taking longer than necessary. +- Fix web databases not respecting lock timeouts. + ## 2024-11-06 ### Changes diff --git a/packages/sqlite_async/CHANGELOG.md b/packages/sqlite_async/CHANGELOG.md index 42a5531..bf1fc83 100644 --- a/packages/sqlite_async/CHANGELOG.md +++ b/packages/sqlite_async/CHANGELOG.md @@ -1,6 +1,9 @@ ## 0.11.5 - - Allow profiling queries. +- Allow profiling queries. Queries are profiled by default in debug and profile builds, the runtime + for queries is added to profiling timelines under the `sqlite_async` tag. +- Fix cancelling `watch()` queries sometimes taking longer than necessary. +- Fix web databases not respecting lock timeouts. ## 0.11.4 From 277dfe03f285fae6ad2d78807f2bfe4c6351baf1 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Tue, 27 May 2025 23:06:01 +0200 Subject: [PATCH 55/90] Fix unrecoverable error when opening native db --- .../lib/src/common/port_channel_native.dart | 4 +-- .../native_sqlite_connection_impl.dart | 23 +++++++++------ .../database/native_sqlite_database.dart | 4 +-- .../sqlite_async/test/native/basic_test.dart | 28 +++++++++++++++++++ 4 files changed, 45 insertions(+), 14 deletions(-) diff --git a/packages/sqlite_async/lib/src/common/port_channel_native.dart b/packages/sqlite_async/lib/src/common/port_channel_native.dart index 8b05feb..bb8318e 100644 --- a/packages/sqlite_async/lib/src/common/port_channel_native.dart +++ b/packages/sqlite_async/lib/src/common/port_channel_native.dart @@ -30,12 +30,10 @@ class ParentPortClient implements PortClient { ParentPortClient() { final initCompleter = Completer.sync(); sendPortFuture = initCompleter.future; - sendPortFuture.then((value) { - sendPort = value; - }); _receivePort.listen((message) { if (message is _InitMessage) { assert(!initCompleter.isCompleted); + sendPort = message.port; initCompleter.complete(message.port); } else if (message is _PortChannelResult) { final handler = handlers.remove(message.requestId); diff --git a/packages/sqlite_async/lib/src/native/database/native_sqlite_connection_impl.dart b/packages/sqlite_async/lib/src/native/database/native_sqlite_connection_impl.dart index ae3c376..38c184e 100644 --- a/packages/sqlite_async/lib/src/native/database/native_sqlite_connection_impl.dart +++ b/packages/sqlite_async/lib/src/native/database/native_sqlite_connection_impl.dart @@ -36,6 +36,7 @@ class SqliteConnectionImpl final bool readOnly; final bool profileQueries; + bool _didOpenSuccessfully = false; SqliteConnectionImpl({ required AbstractDefaultSqliteOpenFactory openFactory, @@ -47,11 +48,11 @@ class SqliteConnectionImpl bool primary = false, }) : _writeMutex = mutex, profileQueries = openFactory.sqliteOptions.profileQueries { - isInitialized = _isolateClient.ready; this.upstreamPort = upstreamPort ?? listenForEvents(); // Accept an incoming stream of updates, or expose one if not given. this.updates = updates ?? updatesController.stream; - _open(openFactory, primary: primary, upstreamPort: this.upstreamPort); + isInitialized = + _open(openFactory, primary: primary, upstreamPort: this.upstreamPort); } Future get ready async { @@ -100,6 +101,7 @@ class SqliteConnectionImpl _isolateClient.tieToIsolate(_isolate); _isolate.resume(_isolate.pauseCapability!); await _isolateClient.ready; + _didOpenSuccessfully = true; }); } @@ -107,15 +109,18 @@ class SqliteConnectionImpl Future close() async { eventsPort?.close(); await _connectionMutex.lock(() async { - if (readOnly) { - await _isolateClient.post(const _SqliteIsolateConnectionClose()); - } else { - // In some cases, disposing a write connection lock the database. - // We use the lock here to avoid "database is locked" errors. - await _writeMutex.lock(() async { + if (_didOpenSuccessfully) { + if (readOnly) { await _isolateClient.post(const _SqliteIsolateConnectionClose()); - }); + } else { + // In some cases, disposing a write connection lock the database. + // We use the lock here to avoid "database is locked" errors. + await _writeMutex.lock(() async { + await _isolateClient.post(const _SqliteIsolateConnectionClose()); + }); + } } + _isolate.kill(); }); } diff --git a/packages/sqlite_async/lib/src/native/database/native_sqlite_database.dart b/packages/sqlite_async/lib/src/native/database/native_sqlite_database.dart index 5cb60f3..7bea111 100644 --- a/packages/sqlite_async/lib/src/native/database/native_sqlite_database.dart +++ b/packages/sqlite_async/lib/src/native/database/native_sqlite_database.dart @@ -34,8 +34,8 @@ class SqliteDatabaseImpl @override @protected - // Native doesn't require any asynchronous initialization - late Future isInitialized = Future.value(); + // ignore: invalid_use_of_protected_member + late Future isInitialized = _internalConnection.isInitialized; late final SqliteConnectionImpl _internalConnection; late final SqliteConnectionPool _pool; diff --git a/packages/sqlite_async/test/native/basic_test.dart b/packages/sqlite_async/test/native/basic_test.dart index eba5493..dec1fed 100644 --- a/packages/sqlite_async/test/native/basic_test.dart +++ b/packages/sqlite_async/test/native/basic_test.dart @@ -344,6 +344,22 @@ void main() { await Future.wait([f1, f2]); }); + + test('reports open error', () async { + // Ensure that a db that fails to open doesn't report any unhandled + // exceptions. This could happen when e.g. SQLCipher is used and the open + // factory supplies a wrong key pragma (because a subsequent pragma to + // change the journal mode then fails with a "not a database" error). + final db = + SqliteDatabase.withFactory(_InvalidPragmaOnOpenFactory(path: path)); + await expectLater( + db.initialize(), + throwsA( + isA().having( + (e) => e.toString(), 'toString()', contains('syntax error')), + ), + ); + }); }); } @@ -351,3 +367,15 @@ void main() { void ignore(Future future) { future.then((_) {}, onError: (_) {}); } + +class _InvalidPragmaOnOpenFactory extends DefaultSqliteOpenFactory { + const _InvalidPragmaOnOpenFactory({required super.path}); + + @override + List pragmaStatements(SqliteOpenOptions options) { + return [ + 'invalid syntax to fail open in test', + ...super.pragmaStatements(options), + ]; + } +} From b130aea638d65299a1f5ce25e85a92ab17981379 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Wed, 28 May 2025 13:47:32 +0200 Subject: [PATCH 56/90] Prepare 0.11.6 release --- CHANGELOG.md | 19 +++++++++++++++++++ packages/sqlite_async/CHANGELOG.md | 5 +++++ packages/sqlite_async/pubspec.yaml | 2 +- 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d23d6d6..8563871 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,25 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## 2025-05-28 + +--- + +Packages with breaking changes: + + - There are no breaking changes in this release. + +Packages with other changes: + + - [`sqlite_async` - `v0.11.6`](#sqlite_async---v0116) + +--- + +#### `sqlite_async` - `v0.11.6` + +- Native: Consistently report errors when opening the database instead of + causing unhandled exceptions. + ## 2025-05-22 --- diff --git a/packages/sqlite_async/CHANGELOG.md b/packages/sqlite_async/CHANGELOG.md index bf1fc83..6f3a51d 100644 --- a/packages/sqlite_async/CHANGELOG.md +++ b/packages/sqlite_async/CHANGELOG.md @@ -1,3 +1,8 @@ +## 0.11.6 + +- Native: Consistently report errors when opening the database instead of + causing unhandled exceptions. + ## 0.11.5 - Allow profiling queries. Queries are profiled by default in debug and profile builds, the runtime diff --git a/packages/sqlite_async/pubspec.yaml b/packages/sqlite_async/pubspec.yaml index 5142162..00f2077 100644 --- a/packages/sqlite_async/pubspec.yaml +++ b/packages/sqlite_async/pubspec.yaml @@ -1,6 +1,6 @@ name: sqlite_async description: High-performance asynchronous interface for SQLite on Dart and Flutter. -version: 0.11.5 +version: 0.11.6 repository: https://github.com/powersync-ja/sqlite_async.dart environment: sdk: ">=3.5.0 <4.0.0" From bec5006fe63d7f0238780807a380bdf74bd33de7 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Mon, 2 Jun 2025 22:04:05 +0200 Subject: [PATCH 57/90] Fix a race condition around OPFS databases --- .../lib/src/web/database/web_sqlite_database.dart | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/packages/sqlite_async/lib/src/web/database/web_sqlite_database.dart b/packages/sqlite_async/lib/src/web/database/web_sqlite_database.dart index 0f38b1c..c6d1b75 100644 --- a/packages/sqlite_async/lib/src/web/database/web_sqlite_database.dart +++ b/packages/sqlite_async/lib/src/web/database/web_sqlite_database.dart @@ -8,7 +8,6 @@ import 'package:sqlite_async/src/common/sqlite_database.dart'; import 'package:sqlite_async/src/sqlite_connection.dart'; import 'package:sqlite_async/src/sqlite_options.dart'; import 'package:sqlite_async/src/update_notification.dart'; -import 'package:sqlite_async/src/web/web_mutex.dart'; import 'package:sqlite_async/src/web/web_sqlite_open_factory.dart'; import 'package:sqlite_async/web.dart'; @@ -43,7 +42,6 @@ class SqliteDatabaseImpl @override AbstractDefaultSqliteOpenFactory openFactory; - late final Mutex mutex; late final WebDatabase _connection; StreamSubscription? _broadcastUpdatesSubscription; @@ -77,15 +75,15 @@ class SqliteDatabaseImpl /// 4. Creating temporary views or triggers. SqliteDatabaseImpl.withFactory(this.openFactory, {this.maxReaders = SqliteDatabase.defaultMaxReaders}) { - mutex = MutexImpl(); // This way the `updates` member is available synchronously updates = updatesController.stream; isInitialized = _init(); } Future _init() async { - _connection = await openFactory.openConnection(SqliteOpenOptions( - primaryConnection: true, readOnly: false, mutex: mutex)) as WebDatabase; + _connection = await openFactory.openConnection( + SqliteOpenOptions(primaryConnection: true, readOnly: false)) + as WebDatabase; final broadcastUpdates = _connection.broadcastUpdates; if (broadcastUpdates == null) { From 1ba26159557b3921751d004063dcd426afba819f Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Tue, 3 Jun 2025 10:16:41 +0200 Subject: [PATCH 58/90] Skip lock timeout test --- packages/sqlite_async/test/basic_test.dart | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/sqlite_async/test/basic_test.dart b/packages/sqlite_async/test/basic_test.dart index ed2f708..fb98e17 100644 --- a/packages/sqlite_async/test/basic_test.dart +++ b/packages/sqlite_async/test/basic_test.dart @@ -229,6 +229,10 @@ void main() { ); await completion; + }, onPlatform: { + 'browser': Skip( + 'Web locks are managed with a shared worker, which does not support timeouts', + ) }); }); } From 9486ae1420e8681047e4a0c355e2c00f93e0883c Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Tue, 3 Jun 2025 10:55:01 +0200 Subject: [PATCH 59/90] Shared worker: Release mutex when tab closes --- packages/sqlite_async/build.yaml | 9 ++++ packages/sqlite_async/example/web/index.html | 29 +++++++++++++ packages/sqlite_async/example/web/main.dart | 41 +++++++++++++++++++ packages/sqlite_async/example/web/worker.dart | 6 +++ .../lib/src/web/worker/worker_utils.dart | 26 ++++++++++++ 5 files changed, 111 insertions(+) create mode 100644 packages/sqlite_async/build.yaml create mode 100644 packages/sqlite_async/example/web/index.html create mode 100644 packages/sqlite_async/example/web/main.dart create mode 100644 packages/sqlite_async/example/web/worker.dart diff --git a/packages/sqlite_async/build.yaml b/packages/sqlite_async/build.yaml new file mode 100644 index 0000000..91e9a3d --- /dev/null +++ b/packages/sqlite_async/build.yaml @@ -0,0 +1,9 @@ +targets: + $default: + builders: + build_web_compilers:entrypoint: + options: + # Workers can't be compiled with dartdevc, so use dart2js for the example + compiler: dart2js + generate_for: + - example/web/** diff --git a/packages/sqlite_async/example/web/index.html b/packages/sqlite_async/example/web/index.html new file mode 100644 index 0000000..4bc69de --- /dev/null +++ b/packages/sqlite_async/example/web/index.html @@ -0,0 +1,29 @@ + + + + + + sqlite_async web demo + + + + + +

sqlite_async demo

+ +
+This page is used to test the sqlite_async package on the web. +Use the console to open and interact with databases. + +
+
+const db = await open('test.db');
+const lock = await write_lock(db);
+release_lock(lock);
+
+
+
+ + + + diff --git a/packages/sqlite_async/example/web/main.dart b/packages/sqlite_async/example/web/main.dart new file mode 100644 index 0000000..95bd1f1 --- /dev/null +++ b/packages/sqlite_async/example/web/main.dart @@ -0,0 +1,41 @@ +import 'dart:async'; +import 'dart:js_interop'; +import 'dart:js_interop_unsafe'; + +import 'package:sqlite_async/sqlite_async.dart'; + +void main() { + globalContext['open'] = (String path) { + return Future(() async { + final db = SqliteDatabase( + path: path, + options: SqliteOptions( + webSqliteOptions: WebSqliteOptions( + wasmUri: + 'https://cdn.jsdelivr.net/npm/@powersync/dart-wasm-bundles@latest/dist/sqlite3.wasm', + workerUri: 'worker.dart.js', + ), + ), + ); + await db.initialize(); + return db.toJSBox; + }).toJS; + }.toJS; + + globalContext['write_lock'] = (JSBoxedDartObject db) { + final hasLock = Completer(); + final completer = Completer(); + + (db.toDart as SqliteDatabase).writeLock((_) async { + print('has write lock!'); + hasLock.complete(); + await completer.future; + }); + + return hasLock.future.then((_) => completer.toJSBox).toJS; + }.toJS; + + globalContext['release_lock'] = (JSBoxedDartObject db) { + (db.toDart as Completer).complete(); + }.toJS; +} diff --git a/packages/sqlite_async/example/web/worker.dart b/packages/sqlite_async/example/web/worker.dart new file mode 100644 index 0000000..481455d --- /dev/null +++ b/packages/sqlite_async/example/web/worker.dart @@ -0,0 +1,6 @@ +import 'package:sqlite_async/sqlite3_web.dart'; +import 'package:sqlite_async/sqlite3_web_worker.dart'; + +void main() { + WebSqlite.workerEntrypoint(controller: AsyncSqliteController()); +} diff --git a/packages/sqlite_async/lib/src/web/worker/worker_utils.dart b/packages/sqlite_async/lib/src/web/worker/worker_utils.dart index af39747..3ecb257 100644 --- a/packages/sqlite_async/lib/src/web/worker/worker_utils.dart +++ b/packages/sqlite_async/lib/src/web/worker/worker_utils.dart @@ -56,9 +56,27 @@ class AsyncSqliteDatabase extends WorkerDatabase { // these requests for shared workers, so we can assume each database is only // opened once and we don't need web locks here. final mutex = ReadWriteMutex(); + final Map _state = {}; AsyncSqliteDatabase({required this.database}); + _ConnectionState _findState(ClientConnection connection) { + return _state.putIfAbsent(connection, _ConnectionState.new); + } + + void _markHoldsMutex(ClientConnection connection) { + final state = _findState(connection); + state.holdsMutex = true; + if (!state.hasOnCloseListener) { + state.hasOnCloseListener = true; + connection.closed.then((_) { + if (state.holdsMutex) { + mutex.release(); + } + }); + } + } + @override Future handleCustomRequest( ClientConnection connection, JSAny? request) async { @@ -67,9 +85,12 @@ class AsyncSqliteDatabase extends WorkerDatabase { switch (message.kind) { case CustomDatabaseMessageKind.requestSharedLock: await mutex.acquireRead(); + _markHoldsMutex(connection); case CustomDatabaseMessageKind.requestExclusiveLock: await mutex.acquireWrite(); + _markHoldsMutex(connection); case CustomDatabaseMessageKind.releaseLock: + _findState(connection).holdsMutex = false; mutex.release(); case CustomDatabaseMessageKind.lockObtained: throw UnsupportedError('This is a response, not a request'); @@ -123,3 +144,8 @@ class AsyncSqliteDatabase extends WorkerDatabase { return resultSetMap; } } + +final class _ConnectionState { + bool hasOnCloseListener = false; + bool holdsMutex = false; +} From cfdb89c74e6da505702dbcc706dd06df5364793e Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Tue, 3 Jun 2025 11:10:24 +0200 Subject: [PATCH 60/90] Actually we want to compile tests too --- packages/sqlite_async/build.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/sqlite_async/build.yaml b/packages/sqlite_async/build.yaml index 91e9a3d..0774cc7 100644 --- a/packages/sqlite_async/build.yaml +++ b/packages/sqlite_async/build.yaml @@ -5,5 +5,3 @@ targets: options: # Workers can't be compiled with dartdevc, so use dart2js for the example compiler: dart2js - generate_for: - - example/web/** From 6c78c7bc7785381070e1f408d16c7f3b44926a77 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Tue, 3 Jun 2025 11:37:22 +0200 Subject: [PATCH 61/90] Prepare release --- CHANGELOG.md | 20 ++++++++++++++++++++ packages/sqlite_async/CHANGELOG.md | 6 ++++++ packages/sqlite_async/pubspec.yaml | 2 +- 3 files changed, 27 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8563871..d9cda07 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,26 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## 2025-06-03 + +--- + +Packages with breaking changes: + + - There are no breaking changes in this release. + +Packages with other changes: + + - [`sqlite_async` - `v0.11.7`](#sqlite_async---v0117) + +--- + +#### `sqlite_async` - `v0.11.7` + +- Shared worker: Release locks owned by connected client tab when it closes. +- Fix web concurrency issues: Consistently apply a shared mutex or let a shared + worker coordinate access. + ## 2025-05-28 --- diff --git a/packages/sqlite_async/CHANGELOG.md b/packages/sqlite_async/CHANGELOG.md index 6f3a51d..97dbf50 100644 --- a/packages/sqlite_async/CHANGELOG.md +++ b/packages/sqlite_async/CHANGELOG.md @@ -1,3 +1,9 @@ +## 0.11.7 + +- Shared worker: Release locks owned by connected client tab when it closes. +- Fix web concurrency issues: Consistently apply a shared mutex or let a shared + worker coordinate access. + ## 0.11.6 - Native: Consistently report errors when opening the database instead of diff --git a/packages/sqlite_async/pubspec.yaml b/packages/sqlite_async/pubspec.yaml index 00f2077..bb27d60 100644 --- a/packages/sqlite_async/pubspec.yaml +++ b/packages/sqlite_async/pubspec.yaml @@ -1,6 +1,6 @@ name: sqlite_async description: High-performance asynchronous interface for SQLite on Dart and Flutter. -version: 0.11.6 +version: 0.11.7 repository: https://github.com/powersync-ja/sqlite_async.dart environment: sdk: ">=3.5.0 <4.0.0" From e5d31b2d17c21c278b54d55608e0e940ef026f35 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Fri, 13 Jun 2025 17:59:49 +0200 Subject: [PATCH 62/90] Support nested transactions --- .../connection/sync_sqlite_connection.dart | 22 ++- .../sqlite_async/lib/src/impl/context.dart | 186 ++++++++++++++++++ .../native_sqlite_connection_impl.dart | 13 +- .../lib/src/sqlite_connection.dart | 22 ++- .../sqlite_async/lib/src/sqlite_queries.dart | 2 +- .../lib/src/utils/shared_utils.dart | 18 -- .../sqlite_async/lib/src/web/database.dart | 67 +++---- 7 files changed, 251 insertions(+), 79 deletions(-) create mode 100644 packages/sqlite_async/lib/src/impl/context.dart diff --git a/packages/sqlite_async/lib/src/common/connection/sync_sqlite_connection.dart b/packages/sqlite_async/lib/src/common/connection/sync_sqlite_connection.dart index d84bdaa..f63e0df 100644 --- a/packages/sqlite_async/lib/src/common/connection/sync_sqlite_connection.dart +++ b/packages/sqlite_async/lib/src/common/connection/sync_sqlite_connection.dart @@ -8,9 +8,11 @@ import 'package:sqlite_async/src/sqlite_queries.dart'; import 'package:sqlite_async/src/update_notification.dart'; import 'package:sqlite_async/src/utils/profiler.dart'; +import '../../impl/context.dart'; + /// A simple "synchronous" connection which provides the async SqliteConnection /// implementation using a synchronous SQLite connection -class SyncSqliteConnection extends SqliteConnection with SqliteQueries { +class SyncSqliteConnection with SqliteQueries implements SqliteConnection { final CommonDatabase db; late Mutex mutex; @override @@ -44,7 +46,10 @@ class SyncSqliteConnection extends SqliteConnection with SqliteQueries { return mutex.lock( () { task?.finish(); - return callback(SyncReadContext(db, parent: task)); + return ScopedReadContext.assumeReadLock( + _UnsafeSyncContext(db, parent: task), + callback, + ); }, timeout: lockTimeout, ); @@ -59,7 +64,10 @@ class SyncSqliteConnection extends SqliteConnection with SqliteQueries { return mutex.lock( () { task?.finish(); - return callback(SyncWriteContext(db, parent: task)); + return ScopedWriteContext.assumeWriteLock( + _UnsafeSyncContext(db, parent: task), + callback, + ); }, timeout: lockTimeout, ); @@ -80,12 +88,12 @@ class SyncSqliteConnection extends SqliteConnection with SqliteQueries { } } -class SyncReadContext implements SqliteReadContext { +final class _UnsafeSyncContext extends UnscopedContext { final TimelineTask? task; CommonDatabase db; - SyncReadContext(this.db, {TimelineTask? parent}) + _UnsafeSyncContext(this.db, {TimelineTask? parent}) : task = TimelineTask(parent: parent); @override @@ -129,10 +137,6 @@ class SyncReadContext implements SqliteReadContext { Future getAutoCommit() async { return db.autocommit; } -} - -class SyncWriteContext extends SyncReadContext implements SqliteWriteContext { - SyncWriteContext(super.db, {super.parent}); @override Future execute(String sql, diff --git a/packages/sqlite_async/lib/src/impl/context.dart b/packages/sqlite_async/lib/src/impl/context.dart new file mode 100644 index 0000000..8a06998 --- /dev/null +++ b/packages/sqlite_async/lib/src/impl/context.dart @@ -0,0 +1,186 @@ +import 'package:sqlite3/common.dart'; + +import '../sqlite_connection.dart'; + +abstract base class UnscopedContext implements SqliteReadContext { + Future execute(String sql, List parameters); + Future executeBatch(String sql, List> parameterSets); + + /// Returns an [UnscopedContext] useful as the outermost transaction. + /// + /// This is called by [ScopedWriteContext.writeTransaction] _after_ executing + /// the first `BEGIN` statement. + /// This is used on the web to assert that the auto-commit state is false + /// before running statements. + UnscopedContext interceptOutermostTransaction() { + return this; + } +} + +final class ScopedReadContext implements SqliteReadContext { + final UnscopedContext _context; + + /// Whether this context view is locked on an inner operation like a + /// transaction. + /// + /// We don't use a mutex because we don't want to serialize access - we just + /// want to forbid concurrent operations. + bool _isLocked = false; + + /// Whether this particular view of a read context has been closed, e.g. + /// because the callback owning it has returned. + bool _closed = false; + + ScopedReadContext(this._context); + + void _checkNotLocked() { + _checkStillOpen(); + + if (_isLocked) { + throw StateError( + 'The context from the callback was locked, e.g. due to a nested ' + 'transaction.'); + } + } + + void _checkStillOpen() { + if (_closed) { + throw StateError('This context to a callback is no longer open. ' + 'Make sure to await all statements on a database to avoid a context ' + 'still being used after its callback has finished.'); + } + } + + @override + bool get closed => _closed || _context.closed; + + @override + Future computeWithDatabase( + Future Function(CommonDatabase db) compute) async { + _checkNotLocked(); + return await _context.computeWithDatabase(compute); + } + + @override + Future get(String sql, [List parameters = const []]) async { + _checkNotLocked(); + final rows = await getAll(sql, parameters); + return rows.first; + } + + @override + Future getAll(String sql, + [List parameters = const []]) async { + _checkNotLocked(); + return await _context.getAll(sql, parameters); + } + + @override + Future getAutoCommit() async { + _checkStillOpen(); + return _context.getAutoCommit(); + } + + @override + Future getOptional(String sql, + [List parameters = const []]) async { + _checkNotLocked(); + final rows = await getAll(sql, parameters); + return rows.firstOrNull; + } + + void invalidate() => _closed = true; + + static Future assumeReadLock( + UnscopedContext unsafe, + Future Function(SqliteReadContext) callback, + ) async { + final scoped = ScopedReadContext(unsafe); + try { + return await callback(scoped); + } finally { + scoped.invalidate(); + } + } +} + +final class ScopedWriteContext extends ScopedReadContext + implements SqliteWriteContext { + /// The "depth" of virtual nested transaction. + /// + /// A value of `0` indicates that this is operating outside of a transaction. + /// A value of `1` indicates a regular transaction (guarded with `BEGIN` and + /// `COMMIT` statements). + /// All higher values indicate a nested transaction implemented with + /// savepoint statements. + final int transactionDepth; + + ScopedWriteContext(super._context, {this.transactionDepth = 0}); + + @override + Future execute(String sql, + [List parameters = const []]) async { + _checkNotLocked(); + return await _context.execute(sql, parameters); + } + + @override + Future executeBatch( + String sql, List> parameterSets) async { + _checkNotLocked(); + + return await _context.executeBatch(sql, parameterSets); + } + + @override + Future writeTransaction( + Future Function(SqliteWriteContext tx) callback) async { + _checkNotLocked(); + final (begin, commit, rollback) = _beginCommitRollback(transactionDepth); + ScopedWriteContext? inner; + + try { + _isLocked = true; + + await _context.execute(begin, const []); + inner = + ScopedWriteContext(_context, transactionDepth: transactionDepth + 1); + final result = await callback(inner); + await _context.execute(commit, const []); + return result; + } catch (e) { + try { + await _context.execute(rollback, const []); + } catch (e) { + // In rare cases, a ROLLBACK may fail. + // Safe to ignore. + } + rethrow; + } finally { + inner?.invalidate(); + } + } + + static (String, String, String) _beginCommitRollback(int level) { + return switch (level) { + 0 => ('BEGIN IMMEDIATE', 'COMMIT', 'ROLLBACK'), + final nested => ( + 'SAVEPOINT s$nested', + 'RELEASE s$nested', + 'ROLLBACK TO s$nested' + ) + }; + } + + static Future assumeWriteLock( + UnscopedContext unsafe, + Future Function(SqliteWriteContext) callback, + ) async { + final scoped = ScopedWriteContext(unsafe); + try { + return await callback(scoped); + } finally { + scoped.invalidate(); + } + } +} diff --git a/packages/sqlite_async/lib/src/native/database/native_sqlite_connection_impl.dart b/packages/sqlite_async/lib/src/native/database/native_sqlite_connection_impl.dart index 38c184e..65f9db8 100644 --- a/packages/sqlite_async/lib/src/native/database/native_sqlite_connection_impl.dart +++ b/packages/sqlite_async/lib/src/native/database/native_sqlite_connection_impl.dart @@ -14,6 +14,7 @@ import 'package:sqlite_async/src/update_notification.dart'; import 'package:sqlite_async/src/utils/profiler.dart'; import 'package:sqlite_async/src/utils/shared_utils.dart'; +import '../../impl/context.dart'; import 'upstream_updates.dart'; typedef TxCallback = Future Function(CommonDatabase db); @@ -64,8 +65,8 @@ class SqliteConnectionImpl return _isolateClient.closed; } - _TransactionContext _context() { - return _TransactionContext( + _UnsafeContext _context() { + return _UnsafeContext( _isolateClient, profileQueries ? TimelineTask() : null); } @@ -137,7 +138,7 @@ class SqliteConnectionImpl return _connectionMutex.lock(() async { final ctx = _context(); try { - return await callback(ctx); + return ScopedReadContext.assumeReadLock(ctx, callback); } finally { await ctx.close(); } @@ -160,7 +161,7 @@ class SqliteConnectionImpl return await _writeMutex.lock(() async { final ctx = _context(); try { - return await callback(ctx); + return ScopedWriteContext.assumeWriteLock(ctx, callback); } finally { await ctx.close(); } @@ -177,14 +178,14 @@ class SqliteConnectionImpl int _nextCtxId = 1; -class _TransactionContext implements SqliteWriteContext { +final class _UnsafeContext extends UnscopedContext { final PortClient _sendPort; bool _closed = false; final int ctxId = _nextCtxId++; final TimelineTask? task; - _TransactionContext(this._sendPort, this.task); + _UnsafeContext(this._sendPort, this.task); @override bool get closed { diff --git a/packages/sqlite_async/lib/src/sqlite_connection.dart b/packages/sqlite_async/lib/src/sqlite_connection.dart index bd2292b..0e360fc 100644 --- a/packages/sqlite_async/lib/src/sqlite_connection.dart +++ b/packages/sqlite_async/lib/src/sqlite_connection.dart @@ -8,7 +8,7 @@ import 'package:sqlite_async/src/update_notification.dart'; import 'common/connection/sync_sqlite_connection.dart'; /// Abstract class representing calls available in a read-only or read-write context. -abstract class SqliteReadContext { +abstract interface class SqliteReadContext { /// Execute a read-only (SELECT) query and return the results. Future getAll(String sql, [List parameters = const []]); @@ -66,7 +66,7 @@ abstract class SqliteReadContext { } /// Abstract class representing calls available in a read-write context. -abstract class SqliteWriteContext extends SqliteReadContext { +abstract interface class SqliteWriteContext extends SqliteReadContext { /// Execute a write query (INSERT, UPDATE, DELETE) and return the results (if any). Future execute(String sql, [List parameters = const []]); @@ -75,13 +75,28 @@ abstract class SqliteWriteContext extends SqliteReadContext { /// parameter set. This is faster than executing separately with each /// parameter set. Future executeBatch(String sql, List> parameterSets); + + /// Open a read-write transaction on this write context. + /// + /// When called on a [SqliteConnection], this takes a global lock - only one + /// write write transaction can execute against the database at a time. This + /// applies even when constructing separate [SqliteDatabase] instances for the + /// same database file. + /// + /// Statements within the transaction must be done on the provided + /// [SqliteWriteContext] - attempting statements on the [SqliteConnection] + /// instance will error. + /// It is forbidden to use the [SqliteWriteContext] after the [callback] + /// completes. + Future writeTransaction( + Future Function(SqliteWriteContext tx) callback); } /// Abstract class representing a connection to the SQLite database. /// /// This package typically pools multiple [SqliteConnection] instances into a /// managed [SqliteDatabase] automatically. -abstract class SqliteConnection extends SqliteWriteContext { +abstract interface class SqliteConnection extends SqliteWriteContext { /// Default constructor for subclasses. SqliteConnection(); @@ -123,6 +138,7 @@ abstract class SqliteConnection extends SqliteWriteContext { /// Statements within the transaction must be done on the provided /// [SqliteWriteContext] - attempting statements on the [SqliteConnection] /// instance will error. + @override Future writeTransaction( Future Function(SqliteWriteContext tx) callback, {Duration? lockTimeout}); diff --git a/packages/sqlite_async/lib/src/sqlite_queries.dart b/packages/sqlite_async/lib/src/sqlite_queries.dart index 80bc568..367d23f 100644 --- a/packages/sqlite_async/lib/src/sqlite_queries.dart +++ b/packages/sqlite_async/lib/src/sqlite_queries.dart @@ -107,7 +107,7 @@ mixin SqliteQueries implements SqliteWriteContext, SqliteConnection { Future Function(SqliteWriteContext tx) callback, {Duration? lockTimeout}) async { return writeLock((ctx) async { - return await internalWriteTransaction(ctx, callback); + return ctx.writeTransaction(callback); }, lockTimeout: lockTimeout, debugContext: 'writeTransaction()'); } diff --git a/packages/sqlite_async/lib/src/utils/shared_utils.dart b/packages/sqlite_async/lib/src/utils/shared_utils.dart index c911bbc..9faf928 100644 --- a/packages/sqlite_async/lib/src/utils/shared_utils.dart +++ b/packages/sqlite_async/lib/src/utils/shared_utils.dart @@ -21,24 +21,6 @@ Future internalReadTransaction(SqliteReadContext ctx, } } -Future internalWriteTransaction(SqliteWriteContext ctx, - Future Function(SqliteWriteContext tx) callback) async { - try { - await ctx.execute('BEGIN IMMEDIATE'); - final result = await callback(ctx); - await ctx.execute('COMMIT'); - return result; - } catch (e) { - try { - await ctx.execute('ROLLBACK'); - } catch (e) { - // In rare cases, a ROLLBACK may fail. - // Safe to ignore. - } - rethrow; - } -} - /// Given a SELECT query, return the tables that the query depends on. Future> getSourceTablesText( SqliteReadContext ctx, String sql) async { diff --git a/packages/sqlite_async/lib/src/web/database.dart b/packages/sqlite_async/lib/src/web/database.dart index cd7e215..0464ca0 100644 --- a/packages/sqlite_async/lib/src/web/database.dart +++ b/packages/sqlite_async/lib/src/web/database.dart @@ -8,9 +8,9 @@ import 'package:sqlite3_web/sqlite3_web.dart'; import 'package:sqlite3_web/protocol_utils.dart' as proto; import 'package:sqlite_async/sqlite_async.dart'; import 'package:sqlite_async/src/utils/profiler.dart'; -import 'package:sqlite_async/src/utils/shared_utils.dart'; import 'package:sqlite_async/src/web/database/broadcast_updates.dart'; import 'package:sqlite_async/web.dart'; +import '../impl/context.dart'; import 'protocol.dart'; import 'web_mutex.dart'; @@ -95,12 +95,8 @@ class WebDatabase {Duration? lockTimeout, String? debugContext}) async { if (_mutex case var mutex?) { return await mutex.lock(timeout: lockTimeout, () async { - final context = _SharedContext(this); - try { - return await callback(context); - } finally { - context.markClosed(); - } + return ScopedReadContext.assumeReadLock( + _UnscopedContext(this), callback); }); } else { // No custom mutex, coordinate locks through shared worker. @@ -108,7 +104,8 @@ class WebDatabase CustomDatabaseMessage(CustomDatabaseMessageKind.requestSharedLock)); try { - return await callback(_SharedContext(this)); + return ScopedReadContext.assumeReadLock( + _UnscopedContext(this), callback); } finally { await _database.customRequest( CustomDatabaseMessage(CustomDatabaseMessageKind.releaseLock)); @@ -125,30 +122,24 @@ class WebDatabase Future Function(SqliteWriteContext tx) callback, {Duration? lockTimeout, bool? flush}) { - return writeLock( - (writeContext) => - internalWriteTransaction(writeContext, (context) async { - // All execute calls done in the callback will be checked for the - // autocommit state - return callback(_ExclusiveTransactionContext(this, writeContext)); - }), + return writeLock((writeContext) { + return ScopedWriteContext.assumeWriteLock( + _UnscopedContext(this), callback); + }, debugContext: 'writeTransaction()', lockTimeout: lockTimeout, flush: flush); } @override - - /// Internal writeLock which intercepts transaction context's to verify auto commit is not active Future writeLock(Future Function(SqliteWriteContext tx) callback, {Duration? lockTimeout, String? debugContext, bool? flush}) async { if (_mutex case var mutex?) { return await mutex.lock(timeout: lockTimeout, () async { - final context = _ExclusiveContext(this); + final context = _UnscopedContext(this); try { - return await callback(context); + return await ScopedWriteContext.assumeWriteLock(context, callback); } finally { - context.markClosed(); if (flush != false) { await this.flush(); } @@ -158,11 +149,10 @@ class WebDatabase // No custom mutex, coordinate locks through shared worker. await _database.customRequest(CustomDatabaseMessage( CustomDatabaseMessageKind.requestExclusiveLock)); - final context = _ExclusiveContext(this); + final context = _UnscopedContext(this); try { - return await callback(context); + return await ScopedWriteContext.assumeWriteLock(context, callback); } finally { - context.markClosed(); if (flush != false) { await this.flush(); } @@ -179,17 +169,16 @@ class WebDatabase } } -class _SharedContext implements SqliteReadContext { +final class _UnscopedContext extends UnscopedContext { final WebDatabase _database; - bool _contextClosed = false; final TimelineTask? _task; - _SharedContext(this._database) + _UnscopedContext(this._database) : _task = _database.profileQueries ? TimelineTask() : null; @override - bool get closed => _contextClosed || _database.closed; + bool get closed => _database.closed; @override Future computeWithDatabase( @@ -230,14 +219,6 @@ class _SharedContext implements SqliteReadContext { return results.firstOrNull; } - void markClosed() { - _contextClosed = true; - } -} - -class _ExclusiveContext extends _SharedContext implements SqliteWriteContext { - _ExclusiveContext(super.database); - @override Future execute(String sql, [List parameters = const []]) { return _task.timeAsync('execute', sql: sql, parameters: parameters, () { @@ -258,15 +239,17 @@ class _ExclusiveContext extends _SharedContext implements SqliteWriteContext { }); }); } -} - -class _ExclusiveTransactionContext extends _ExclusiveContext { - SqliteWriteContext baseContext; - - _ExclusiveTransactionContext(super.database, this.baseContext); @override - bool get closed => baseContext.closed; + UnscopedContext interceptOutermostTransaction() { + // All execute calls done in the callback will be checked for the + // autocommit state + return _ExclusiveTransactionContext(_database); + } +} + +final class _ExclusiveTransactionContext extends _UnscopedContext { + _ExclusiveTransactionContext(super._database); Future _executeInternal( String sql, List parameters) async { From 8212a2323931e6738044b5a2d82c8124be3e9af0 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Mon, 16 Jun 2025 11:44:29 +0200 Subject: [PATCH 63/90] Fix tests --- packages/sqlite_async/lib/src/impl/context.dart | 13 +++++++++---- .../database/native_sqlite_connection_impl.dart | 4 ++-- packages/sqlite_async/lib/src/web/database.dart | 10 +++++++--- packages/sqlite_async/test/basic_test.dart | 2 +- 4 files changed, 19 insertions(+), 10 deletions(-) diff --git a/packages/sqlite_async/lib/src/impl/context.dart b/packages/sqlite_async/lib/src/impl/context.dart index 8a06998..e568b33 100644 --- a/packages/sqlite_async/lib/src/impl/context.dart +++ b/packages/sqlite_async/lib/src/impl/context.dart @@ -139,18 +139,23 @@ final class ScopedWriteContext extends ScopedReadContext final (begin, commit, rollback) = _beginCommitRollback(transactionDepth); ScopedWriteContext? inner; + final innerContext = transactionDepth == 0 + ? _context.interceptOutermostTransaction() + : _context; + try { _isLocked = true; await _context.execute(begin, const []); - inner = - ScopedWriteContext(_context, transactionDepth: transactionDepth + 1); + + inner = ScopedWriteContext(innerContext, + transactionDepth: transactionDepth + 1); final result = await callback(inner); - await _context.execute(commit, const []); + await innerContext.execute(commit, const []); return result; } catch (e) { try { - await _context.execute(rollback, const []); + await innerContext.execute(rollback, const []); } catch (e) { // In rare cases, a ROLLBACK may fail. // Safe to ignore. diff --git a/packages/sqlite_async/lib/src/native/database/native_sqlite_connection_impl.dart b/packages/sqlite_async/lib/src/native/database/native_sqlite_connection_impl.dart index 65f9db8..e2df0f3 100644 --- a/packages/sqlite_async/lib/src/native/database/native_sqlite_connection_impl.dart +++ b/packages/sqlite_async/lib/src/native/database/native_sqlite_connection_impl.dart @@ -138,7 +138,7 @@ class SqliteConnectionImpl return _connectionMutex.lock(() async { final ctx = _context(); try { - return ScopedReadContext.assumeReadLock(ctx, callback); + return await ScopedReadContext.assumeReadLock(ctx, callback); } finally { await ctx.close(); } @@ -161,7 +161,7 @@ class SqliteConnectionImpl return await _writeMutex.lock(() async { final ctx = _context(); try { - return ScopedWriteContext.assumeWriteLock(ctx, callback); + return await ScopedWriteContext.assumeWriteLock(ctx, callback); } finally { await ctx.close(); } diff --git a/packages/sqlite_async/lib/src/web/database.dart b/packages/sqlite_async/lib/src/web/database.dart index 0464ca0..3e0797b 100644 --- a/packages/sqlite_async/lib/src/web/database.dart +++ b/packages/sqlite_async/lib/src/web/database.dart @@ -94,7 +94,7 @@ class WebDatabase Future readLock(Future Function(SqliteReadContext tx) callback, {Duration? lockTimeout, String? debugContext}) async { if (_mutex case var mutex?) { - return await mutex.lock(timeout: lockTimeout, () async { + return await mutex.lock(timeout: lockTimeout, () { return ScopedReadContext.assumeReadLock( _UnscopedContext(this), callback); }); @@ -104,7 +104,7 @@ class WebDatabase CustomDatabaseMessage(CustomDatabaseMessageKind.requestSharedLock)); try { - return ScopedReadContext.assumeReadLock( + return await ScopedReadContext.assumeReadLock( _UnscopedContext(this), callback); } finally { await _database.customRequest( @@ -124,7 +124,11 @@ class WebDatabase bool? flush}) { return writeLock((writeContext) { return ScopedWriteContext.assumeWriteLock( - _UnscopedContext(this), callback); + _UnscopedContext(this), + (ctx) async { + return await ctx.writeTransaction(callback); + }, + ); }, debugContext: 'writeTransaction()', lockTimeout: lockTimeout, diff --git a/packages/sqlite_async/test/basic_test.dart b/packages/sqlite_async/test/basic_test.dart index fb98e17..4bbd86b 100644 --- a/packages/sqlite_async/test/basic_test.dart +++ b/packages/sqlite_async/test/basic_test.dart @@ -122,7 +122,7 @@ void main() { ['Test Data']); expect(rs.rows[0], equals(['Test Data'])); }); - expect(await savedTx!.getAutoCommit(), equals(true)); + expect(await db.getAutoCommit(), equals(true)); expect(savedTx!.closed, equals(true)); }); From d15545911878a20d2358782e48e621d3ce00d5e2 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Mon, 16 Jun 2025 12:00:14 +0200 Subject: [PATCH 64/90] Add tests --- .../sqlite_async/lib/src/impl/context.dart | 1 + packages/sqlite_async/test/basic_test.dart | 67 +++++++++++++++++++ 2 files changed, 68 insertions(+) diff --git a/packages/sqlite_async/lib/src/impl/context.dart b/packages/sqlite_async/lib/src/impl/context.dart index e568b33..afc2ba5 100644 --- a/packages/sqlite_async/lib/src/impl/context.dart +++ b/packages/sqlite_async/lib/src/impl/context.dart @@ -162,6 +162,7 @@ final class ScopedWriteContext extends ScopedReadContext } rethrow; } finally { + _isLocked = false; inner?.invalidate(); } } diff --git a/packages/sqlite_async/test/basic_test.dart b/packages/sqlite_async/test/basic_test.dart index 4bbd86b..6a315da 100644 --- a/packages/sqlite_async/test/basic_test.dart +++ b/packages/sqlite_async/test/basic_test.dart @@ -189,6 +189,73 @@ void main() { ); }); + group('nested transaction', () { + const insert = 'INSERT INTO test_data (description) VALUES(?);'; + late SqliteDatabase db; + + setUp(() async { + db = await testUtils.setupDatabase(path: path); + await createTables(db); + }); + + tearDown(() => db.close()); + + test('run in outer transaction', () async { + await db.writeTransaction((tx) async { + await tx.execute(insert, ['first']); + + await tx.writeTransaction((tx) async { + await tx.execute(insert, ['second']); + }); + + expect(await tx.getAll('SELECT * FROM test_data'), hasLength(2)); + }); + + expect(await db.getAll('SELECT * FROM test_data'), hasLength(2)); + }); + + test('can rollback inner transaction', () async { + await db.writeTransaction((tx) async { + await tx.execute(insert, ['first']); + + await tx.writeTransaction((tx) async { + await tx.execute(insert, ['second']); + }); + + await expectLater(() async { + await tx.writeTransaction((tx) async { + await tx.execute(insert, ['third']); + expect(await tx.getAll('SELECT * FROM test_data'), hasLength(3)); + throw 'rollback please'; + }); + }, throwsA(anything)); + + expect(await tx.getAll('SELECT * FROM test_data'), hasLength(2)); + }); + + expect(await db.getAll('SELECT * FROM test_data'), hasLength(2)); + }); + + test('cannot use outer transaction while inner is active', () async { + await db.writeTransaction((outer) async { + await outer.writeTransaction((inner) async { + await expectLater(outer.execute('SELECT 1'), throwsStateError); + }); + }); + }); + + test('cannot use inner after leaving scope', () async { + await db.writeTransaction((tx) async { + late SqliteWriteContext inner; + await tx.writeTransaction((tx) async { + inner = tx; + }); + + await expectLater(inner.execute('SELECT 1'), throwsStateError); + }); + }); + }); + test('can use raw database instance', () async { final factory = await testUtils.testFactory(); final raw = await factory.openDatabaseForSingleConnection(); From 97bc2c83484b1a6ab6c7f929c666dbaae24f996c Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Mon, 16 Jun 2025 14:07:54 +0200 Subject: [PATCH 65/90] Docs --- .../sqlite_async/lib/src/impl/context.dart | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/packages/sqlite_async/lib/src/impl/context.dart b/packages/sqlite_async/lib/src/impl/context.dart index afc2ba5..548681e 100644 --- a/packages/sqlite_async/lib/src/impl/context.dart +++ b/packages/sqlite_async/lib/src/impl/context.dart @@ -2,6 +2,13 @@ import 'package:sqlite3/common.dart'; import '../sqlite_connection.dart'; +/// A context that can be used to run both reading and writing queries - +/// basically a [SqliteWriteContext] without the ability to start transactions. +/// +/// Instances of this are not given out to clients - instead, they are wrapped +/// with [ScopedReadContext] and [ScopedWriteContext] after obtaining a lock. +/// Those wrapped views have a shorter lifetime (they can be closed +/// independently, and verify that they're not being used after being closed). abstract base class UnscopedContext implements SqliteReadContext { Future execute(String sql, List parameters); Future executeBatch(String sql, List> parameterSets); @@ -17,6 +24,7 @@ abstract base class UnscopedContext implements SqliteReadContext { } } +/// A view over an [UnscopedContext] implementing [SqliteReadContext]. final class ScopedReadContext implements SqliteReadContext { final UnscopedContext _context; @@ -91,6 +99,11 @@ final class ScopedReadContext implements SqliteReadContext { void invalidate() => _closed = true; + /// Creates a short-lived wrapper around the [unsafe] context to safely give + /// [callback] read-access to the database. + /// + /// Assumes that a read lock providing sound access to the inner + /// [UnscopedContext] is held until this future returns. static Future assumeReadLock( UnscopedContext unsafe, Future Function(SqliteReadContext) callback, @@ -178,6 +191,11 @@ final class ScopedWriteContext extends ScopedReadContext }; } + /// Creates a short-lived wrapper around the [unsafe] context to safely give + /// [callback] access to the database. + /// + /// Assumes that a write lock providing sound access to the inner + /// [UnscopedContext] is held until this future returns. static Future assumeWriteLock( UnscopedContext unsafe, Future Function(SqliteWriteContext) callback, From daa5e7a5bef3f91f9075f7ecb9b2054085fd8ce4 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Mon, 16 Jun 2025 14:14:58 +0200 Subject: [PATCH 66/90] Forward get and getOptional calls --- packages/sqlite_async/lib/src/impl/context.dart | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/packages/sqlite_async/lib/src/impl/context.dart b/packages/sqlite_async/lib/src/impl/context.dart index 548681e..0e3eef6 100644 --- a/packages/sqlite_async/lib/src/impl/context.dart +++ b/packages/sqlite_async/lib/src/impl/context.dart @@ -72,8 +72,7 @@ final class ScopedReadContext implements SqliteReadContext { @override Future get(String sql, [List parameters = const []]) async { _checkNotLocked(); - final rows = await getAll(sql, parameters); - return rows.first; + return _context.get(sql, parameters); } @override @@ -93,8 +92,7 @@ final class ScopedReadContext implements SqliteReadContext { Future getOptional(String sql, [List parameters = const []]) async { _checkNotLocked(); - final rows = await getAll(sql, parameters); - return rows.firstOrNull; + return _context.getOptional(sql, parameters); } void invalidate() => _closed = true; From 4b5607625b1d3f37d6f1610ec562f7be2a79ec86 Mon Sep 17 00:00:00 2001 From: Jorge Sardina Date: Fri, 20 Jun 2025 12:52:04 +0200 Subject: [PATCH 67/90] Allow transforming table updates from sqlite_async. --- packages/drift_sqlite_async/CHANGELOG.md | 4 ++++ .../lib/src/connection.dart | 20 ++++++++++++++----- packages/drift_sqlite_async/pubspec.yaml | 2 +- 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/packages/drift_sqlite_async/CHANGELOG.md b/packages/drift_sqlite_async/CHANGELOG.md index b101ac4..fa769a1 100644 --- a/packages/drift_sqlite_async/CHANGELOG.md +++ b/packages/drift_sqlite_async/CHANGELOG.md @@ -1,3 +1,7 @@ +## 0.2.3 + +- Allow transforming table updates from sqlite_async. + ## 0.2.2 - Fix write detection when using UPDATE/INSERT/DELETE with RETURNING in raw queries. diff --git a/packages/drift_sqlite_async/lib/src/connection.dart b/packages/drift_sqlite_async/lib/src/connection.dart index a1af55a..bcde1bf 100644 --- a/packages/drift_sqlite_async/lib/src/connection.dart +++ b/packages/drift_sqlite_async/lib/src/connection.dart @@ -15,12 +15,22 @@ import 'package:sqlite_async/sqlite_async.dart'; class SqliteAsyncDriftConnection extends DatabaseConnection { late StreamSubscription _updateSubscription; - SqliteAsyncDriftConnection(SqliteConnection db, {bool logStatements = false}) - : super(SqliteAsyncQueryExecutor(db, logStatements: logStatements)) { + SqliteAsyncDriftConnection( + SqliteConnection db, { + bool logStatements = false, + Set Function(UpdateNotification)? transformTableUpdate, + }) : super(SqliteAsyncQueryExecutor(db, logStatements: logStatements)) { _updateSubscription = (db as SqliteQueries).updates!.listen((event) { - var setUpdates = {}; - for (var tableName in event.tables) { - setUpdates.add(TableUpdate(tableName)); + final Set setUpdates; + // This is useful to map local table names from PowerSync that are backed by a view name + // which is the entity that the user interacts with. + if (transformTableUpdate != null) { + setUpdates = transformTableUpdate(event); + } else { + setUpdates = {}; + for (var tableName in event.tables) { + setUpdates.add(TableUpdate(tableName)); + } } super.streamQueries.handleTableUpdates(setUpdates); }); diff --git a/packages/drift_sqlite_async/pubspec.yaml b/packages/drift_sqlite_async/pubspec.yaml index 62a64da..c17e833 100644 --- a/packages/drift_sqlite_async/pubspec.yaml +++ b/packages/drift_sqlite_async/pubspec.yaml @@ -1,5 +1,5 @@ name: drift_sqlite_async -version: 0.2.2 +version: 0.2.3 homepage: https://github.com/powersync-ja/sqlite_async.dart repository: https://github.com/powersync-ja/sqlite_async.dart description: Use Drift with a sqlite_async database, allowing both to be used in the same application. From a2b6c4cd2226f867569b03d25318b87d6aea6fde Mon Sep 17 00:00:00 2001 From: David Martos Date: Sun, 22 Jun 2025 12:35:16 +0200 Subject: [PATCH 68/90] typo --- packages/drift_sqlite_async/lib/src/connection.dart | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/drift_sqlite_async/lib/src/connection.dart b/packages/drift_sqlite_async/lib/src/connection.dart index bcde1bf..fa56e2e 100644 --- a/packages/drift_sqlite_async/lib/src/connection.dart +++ b/packages/drift_sqlite_async/lib/src/connection.dart @@ -18,14 +18,14 @@ class SqliteAsyncDriftConnection extends DatabaseConnection { SqliteAsyncDriftConnection( SqliteConnection db, { bool logStatements = false, - Set Function(UpdateNotification)? transformTableUpdate, + Set Function(UpdateNotification)? transformTableUpdates, }) : super(SqliteAsyncQueryExecutor(db, logStatements: logStatements)) { _updateSubscription = (db as SqliteQueries).updates!.listen((event) { final Set setUpdates; // This is useful to map local table names from PowerSync that are backed by a view name // which is the entity that the user interacts with. - if (transformTableUpdate != null) { - setUpdates = transformTableUpdate(event); + if (transformTableUpdates != null) { + setUpdates = transformTableUpdates(event); } else { setUpdates = {}; for (var tableName in event.tables) { From 6e51f5dec53947909869fc7b889712f9f62c81a1 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Tue, 29 Jul 2025 16:41:49 +0200 Subject: [PATCH 69/90] Support 3.8.0 of `package:sqlite3`. --- .../web/worker/throttled_common_database.dart | 28 +++++++++++-------- packages/sqlite_async/pubspec.yaml | 2 +- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/packages/sqlite_async/lib/src/web/worker/throttled_common_database.dart b/packages/sqlite_async/lib/src/web/worker/throttled_common_database.dart index 5f33b6d..8a0e901 100644 --- a/packages/sqlite_async/lib/src/web/worker/throttled_common_database.dart +++ b/packages/sqlite_async/lib/src/web/worker/throttled_common_database.dart @@ -28,12 +28,14 @@ class ThrottledCommonDatabase extends CommonDatabase { DatabaseConfig get config => _db.config; @override - void createAggregateFunction( - {required String functionName, - required AggregateFunction function, - AllowedArgumentCount argumentCount = const AllowedArgumentCount.any(), - bool deterministic = false, - bool directOnly = true}) { + void createAggregateFunction({ + required String functionName, + required AggregateFunction function, + AllowedArgumentCount argumentCount = const AllowedArgumentCount.any(), + bool deterministic = false, + bool directOnly = true, + bool subtype = false, + }) { _db.createAggregateFunction(functionName: functionName, function: function); } @@ -44,12 +46,14 @@ class ThrottledCommonDatabase extends CommonDatabase { } @override - void createFunction( - {required String functionName, - required ScalarFunction function, - AllowedArgumentCount argumentCount = const AllowedArgumentCount.any(), - bool deterministic = false, - bool directOnly = true}) { + void createFunction({ + required String functionName, + required ScalarFunction function, + AllowedArgumentCount argumentCount = const AllowedArgumentCount.any(), + bool deterministic = false, + bool directOnly = true, + bool subtype = false, + }) { _db.createFunction(functionName: functionName, function: function); } diff --git a/packages/sqlite_async/pubspec.yaml b/packages/sqlite_async/pubspec.yaml index bb27d60..e610d7f 100644 --- a/packages/sqlite_async/pubspec.yaml +++ b/packages/sqlite_async/pubspec.yaml @@ -12,7 +12,7 @@ topics: - flutter dependencies: - sqlite3: ^2.7.2 + sqlite3: ^2.8.0 sqlite3_web: ^0.3.0 async: ^2.10.0 collection: ^1.17.0 From 40243bf761ecc0b5268edab6a28408f5bd44b4ff Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Tue, 15 Jul 2025 17:42:48 +0200 Subject: [PATCH 70/90] Drift: Support nested transactions --- packages/drift_sqlite_async/CHANGELOG.md | 4 +++ .../drift_sqlite_async/lib/src/executor.dart | 17 ++++++++++- packages/drift_sqlite_async/pubspec.yaml | 6 ++-- packages/drift_sqlite_async/test/db_test.dart | 30 +++++++++++++++++++ 4 files changed, 53 insertions(+), 4 deletions(-) diff --git a/packages/drift_sqlite_async/CHANGELOG.md b/packages/drift_sqlite_async/CHANGELOG.md index b101ac4..e70cd40 100644 --- a/packages/drift_sqlite_async/CHANGELOG.md +++ b/packages/drift_sqlite_async/CHANGELOG.md @@ -1,3 +1,7 @@ +## 0.2.3 + +- Support nested transactions. + ## 0.2.2 - Fix write detection when using UPDATE/INSERT/DELETE with RETURNING in raw queries. diff --git a/packages/drift_sqlite_async/lib/src/executor.dart b/packages/drift_sqlite_async/lib/src/executor.dart index 41886f3..127462e 100644 --- a/packages/drift_sqlite_async/lib/src/executor.dart +++ b/packages/drift_sqlite_async/lib/src/executor.dart @@ -129,13 +129,28 @@ class _SqliteAsyncTransactionDelegate extends SupportedTransactionDelegate { _SqliteAsyncTransactionDelegate(this._db); + @override + FutureOr Function(QueryDelegate, Future Function(QueryDelegate))? + get startNested => _startNested; + @override Future startTransaction(Future Function(QueryDelegate p1) run) async { - await _db.writeTransaction((context) async { + await _startTransaction(_db, run); + } + + Future _startTransaction( + SqliteWriteContext context, Future Function(QueryDelegate p1) run) async { + await context.writeTransaction((context) async { final delegate = _SqliteAsyncQueryDelegate(context, null); return run(delegate); }); } + + Future _startNested( + QueryDelegate outer, Future Function(QueryDelegate) block) async { + await _startTransaction( + (outer as _SqliteAsyncQueryDelegate)._context, block); + } } class _SqliteAsyncVersionDelegate extends DynamicVersionDelegate { diff --git a/packages/drift_sqlite_async/pubspec.yaml b/packages/drift_sqlite_async/pubspec.yaml index 62a64da..aea987f 100644 --- a/packages/drift_sqlite_async/pubspec.yaml +++ b/packages/drift_sqlite_async/pubspec.yaml @@ -1,5 +1,5 @@ name: drift_sqlite_async -version: 0.2.2 +version: 0.2.3 homepage: https://github.com/powersync-ja/sqlite_async.dart repository: https://github.com/powersync-ja/sqlite_async.dart description: Use Drift with a sqlite_async database, allowing both to be used in the same application. @@ -14,12 +14,12 @@ topics: environment: sdk: ">=3.0.0 <4.0.0" dependencies: - drift: ">=2.19.0 <3.0.0" + drift: ">=2.28.0 <3.0.0" sqlite_async: ^0.11.0 dev_dependencies: build_runner: ^2.4.8 - drift_dev: ">=2.19.0 <3.0.0" + drift_dev: ">=2.28.0 <3.0.0" glob: ^2.1.2 lints: ^5.0.0 sqlite3: ^2.4.0 diff --git a/packages/drift_sqlite_async/test/db_test.dart b/packages/drift_sqlite_async/test/db_test.dart index bda09df..f5b269a 100644 --- a/packages/drift_sqlite_async/test/db_test.dart +++ b/packages/drift_sqlite_async/test/db_test.dart @@ -117,5 +117,35 @@ void main() { final deleted = await dbu.delete(dbu.todoItems).go(); expect(deleted, 10); }); + + test('nested transactions', () async { + await dbu + .into(dbu.todoItems) + .insert(TodoItemsCompanion.insert(description: 'root')); + + await dbu.transaction(() async { + await dbu + .into(dbu.todoItems) + .insert(TodoItemsCompanion.insert(description: 'tx0')); + + await dbu.transaction(() async { + await dbu + .into(dbu.todoItems) + .insert(TodoItemsCompanion.insert(description: 'tx1')); + + await expectLater(() { + return dbu.transaction(() async { + await dbu + .into(dbu.todoItems) + .insert(TodoItemsCompanion.insert(description: 'tx2')); + throw 'rollback'; + }); + }, throwsA(anything)); + }); + }); + + final items = await dbu.todoItems.all().get(); + expect(items.map((e) => e.description).toSet(), {'root', 'tx0', 'tx1'}); + }); }); } From 1f720565467cd16400c4c97e871d70947fb1ea33 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Tue, 15 Jul 2025 17:47:12 +0200 Subject: [PATCH 71/90] Prepare for release --- packages/drift_sqlite_async/pubspec.yaml | 2 +- packages/sqlite_async/CHANGELOG.md | 4 ++++ packages/sqlite_async/pubspec.yaml | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/drift_sqlite_async/pubspec.yaml b/packages/drift_sqlite_async/pubspec.yaml index aea987f..77f250d 100644 --- a/packages/drift_sqlite_async/pubspec.yaml +++ b/packages/drift_sqlite_async/pubspec.yaml @@ -15,7 +15,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: drift: ">=2.28.0 <3.0.0" - sqlite_async: ^0.11.0 + sqlite_async: ^0.11.8 dev_dependencies: build_runner: ^2.4.8 diff --git a/packages/sqlite_async/CHANGELOG.md b/packages/sqlite_async/CHANGELOG.md index 97dbf50..eedfb93 100644 --- a/packages/sqlite_async/CHANGELOG.md +++ b/packages/sqlite_async/CHANGELOG.md @@ -1,3 +1,7 @@ +## 0.11.8 + +- Support nested transactions (emulated with `SAVEPOINT` statements). + ## 0.11.7 - Shared worker: Release locks owned by connected client tab when it closes. diff --git a/packages/sqlite_async/pubspec.yaml b/packages/sqlite_async/pubspec.yaml index e610d7f..a155f2b 100644 --- a/packages/sqlite_async/pubspec.yaml +++ b/packages/sqlite_async/pubspec.yaml @@ -1,6 +1,6 @@ name: sqlite_async description: High-performance asynchronous interface for SQLite on Dart and Flutter. -version: 0.11.7 +version: 0.11.8 repository: https://github.com/powersync-ja/sqlite_async.dart environment: sdk: ">=3.5.0 <4.0.0" From 17012edf88a498e0c04255dbb4e0f6a2d054c299 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Tue, 29 Jul 2025 17:29:02 +0200 Subject: [PATCH 72/90] Update changelog --- CHANGELOG.md | 24 ++++++++++++++++++++++++ packages/sqlite_async/CHANGELOG.md | 1 + 2 files changed, 25 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d9cda07..cafde71 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,30 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## 2025-07-29 + +--- + +Packages with breaking changes: + + - There are no breaking changes in this release. + +Packages with other changes: + + - [`sqlite_async` - `v0.11.8`](#sqlite_async---v0118) + - [`drift_sqlite_async` - `v0.2.3`](#drift_sqlite_async---v023) + +--- + +#### `sqlite_async` - `v0.11.8` + +- Support nested transactions (emulated with `SAVEPOINT` statements). +- Fix web compilation issues with version `2.8.0` of `package:sqlite3`. + +#### `drift_sqlite_async` - `v0.2.3` + +- Support nested transactions. + ## 2025-06-03 --- diff --git a/packages/sqlite_async/CHANGELOG.md b/packages/sqlite_async/CHANGELOG.md index eedfb93..387c944 100644 --- a/packages/sqlite_async/CHANGELOG.md +++ b/packages/sqlite_async/CHANGELOG.md @@ -1,6 +1,7 @@ ## 0.11.8 - Support nested transactions (emulated with `SAVEPOINT` statements). +- Fix web compilation issues with version `2.8.0` of `package:sqlite3`. ## 0.11.7 From 7e0a0c74a89b7481e42a882f2b0fdc97122b684c Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Thu, 7 Aug 2025 10:30:30 +0200 Subject: [PATCH 73/90] Refactor update streams on web --- .../lib/src/utils/shared_utils.dart | 69 +++++++ .../sqlite_async/lib/src/web/database.dart | 8 +- .../sqlite_async/lib/src/web/protocol.dart | 2 + .../lib/src/web/update_notifications.dart | 57 ++++++ .../lib/src/web/web_sqlite_open_factory.dart | 14 +- .../web/worker/throttled_common_database.dart | 191 ------------------ .../lib/src/web/worker/worker_utils.dart | 48 ++++- packages/sqlite_async/lib/web.dart | 25 ++- 8 files changed, 205 insertions(+), 209 deletions(-) create mode 100644 packages/sqlite_async/lib/src/web/update_notifications.dart delete mode 100644 packages/sqlite_async/lib/src/web/worker/throttled_common_database.dart diff --git a/packages/sqlite_async/lib/src/utils/shared_utils.dart b/packages/sqlite_async/lib/src/utils/shared_utils.dart index 9faf928..ff291f4 100644 --- a/packages/sqlite_async/lib/src/utils/shared_utils.dart +++ b/packages/sqlite_async/lib/src/utils/shared_utils.dart @@ -1,6 +1,8 @@ import 'dart:async'; import 'dart:convert'; +import 'package:sqlite3/common.dart'; + import '../sqlite_connection.dart'; Future internalReadTransaction(SqliteReadContext ctx, @@ -75,3 +77,70 @@ Object? mapParameter(Object? parameter) { List mapParameters(List parameters) { return [for (var p in parameters) mapParameter(p)]; } + +extension ThrottledUpdates on CommonDatabase { + /// Wraps [updatesSync] to: + /// + /// - Not fire in transactions. + /// - Fire asynchronously. + /// - Only report table names, which are buffered to avoid duplicates. + Stream> get throttledUpdatedTables { + StreamController>? controller; + var pendingUpdates = {}; + var paused = false; + + Timer? updateDebouncer; + + void maybeFireUpdates() { + updateDebouncer?.cancel(); + updateDebouncer = null; + + if (paused) { + // Continue collecting updates, but don't fire any + return; + } + + if (!autocommit) { + // Inside a transaction - do not fire updates + return; + } + + if (pendingUpdates.isNotEmpty) { + controller!.add(pendingUpdates); + pendingUpdates = {}; + } + } + + void collectUpdate(SqliteUpdate event) { + pendingUpdates.add(event.tableName); + + updateDebouncer ??= + Timer(const Duration(milliseconds: 1), maybeFireUpdates); + } + + StreamSubscription? txSubscription; + StreamSubscription? sourceSubscription; + + controller = StreamController(onListen: () { + txSubscription = commits.listen((_) { + maybeFireUpdates(); + }, onError: (error) { + controller?.addError(error); + }); + + sourceSubscription = updatesSync.listen(collectUpdate, onError: (error) { + controller?.addError(error); + }); + }, onPause: () { + paused = true; + }, onResume: () { + paused = false; + maybeFireUpdates(); + }, onCancel: () { + txSubscription?.cancel(); + sourceSubscription?.cancel(); + }); + + return controller.stream; + } +} diff --git a/packages/sqlite_async/lib/src/web/database.dart b/packages/sqlite_async/lib/src/web/database.dart index 3e0797b..cfaf987 100644 --- a/packages/sqlite_async/lib/src/web/database.dart +++ b/packages/sqlite_async/lib/src/web/database.dart @@ -21,6 +21,9 @@ class WebDatabase final Mutex? _mutex; final bool profileQueries; + @override + final Stream updates; + /// For persistent databases that aren't backed by a shared worker, we use /// web broadcast channels to forward local update events to other tabs. final BroadcastUpdates? broadcastUpdates; @@ -32,6 +35,7 @@ class WebDatabase this._database, this._mutex, { required this.profileQueries, + required this.updates, this.broadcastUpdates, }); @@ -113,10 +117,6 @@ class WebDatabase } } - @override - Stream get updates => - _database.updates.map((event) => UpdateNotification({event.tableName})); - @override Future writeTransaction( Future Function(SqliteWriteContext tx) callback, diff --git a/packages/sqlite_async/lib/src/web/protocol.dart b/packages/sqlite_async/lib/src/web/protocol.dart index cb3a5fd..d17c06b 100644 --- a/packages/sqlite_async/lib/src/web/protocol.dart +++ b/packages/sqlite_async/lib/src/web/protocol.dart @@ -13,6 +13,8 @@ enum CustomDatabaseMessageKind { getAutoCommit, executeInTransaction, executeBatchInTransaction, + updateSubscriptionManagement, + notifyUpdates, } extension type CustomDatabaseMessage._raw(JSObject _) implements JSObject { diff --git a/packages/sqlite_async/lib/src/web/update_notifications.dart b/packages/sqlite_async/lib/src/web/update_notifications.dart new file mode 100644 index 0000000..ecea60d --- /dev/null +++ b/packages/sqlite_async/lib/src/web/update_notifications.dart @@ -0,0 +1,57 @@ +import 'dart:async'; +import 'dart:js_interop'; + +import 'package:sqlite3_web/sqlite3_web.dart'; + +import '../update_notification.dart'; +import 'protocol.dart'; + +/// Utility to request a stream of update notifications from the worker. +/// +/// Because we want to debounce update notifications on the worker, we're using +/// custom requests instead of the default [Database.updates] stream. +/// +/// Clients send a message to the worker to subscribe or unsubscribe, providing +/// an id for the subscription. The worker distributes update notifications with +/// custom requests to the client, which [handleRequest] distributes to the +/// original streams. +final class UpdateNotificationStreams { + var _idCounter = 0; + final Map> _updates = {}; + + Future handleRequest(JSAny? request) async { + final customRequest = request as CustomDatabaseMessage; + if (customRequest.kind == CustomDatabaseMessageKind.notifyUpdates) { + final notification = UpdateNotification(customRequest.rawParameters.toDart + .map((e) => (e as JSString).toDart) + .toSet()); + + _updates[customRequest.rawSql.toDart]?.add(notification); + } + + return null; + } + + Stream updatesFor(Database database) { + final id = (_idCounter++).toString(); + final controller = _updates[id] = StreamController(); + + controller + ..onListen = () { + database.customRequest(CustomDatabaseMessage( + CustomDatabaseMessageKind.updateSubscriptionManagement, + id, + [true], + )); + } + ..onCancel = () { + database.customRequest(CustomDatabaseMessage( + CustomDatabaseMessageKind.updateSubscriptionManagement, + id, + [false], + )); + }; + + return controller.stream; + } +} diff --git a/packages/sqlite_async/lib/src/web/web_sqlite_open_factory.dart b/packages/sqlite_async/lib/src/web/web_sqlite_open_factory.dart index a724329..205734f 100644 --- a/packages/sqlite_async/lib/src/web/web_sqlite_open_factory.dart +++ b/packages/sqlite_async/lib/src/web/web_sqlite_open_factory.dart @@ -69,15 +69,19 @@ class DefaultSqliteOpenFactory ? null : MutexImpl(identifier: path); // Use the DB path as a mutex identifier - BroadcastUpdates? updates; + BroadcastUpdates? broadcastUpdates; if (connection.access != AccessMode.throughSharedWorker && connection.storage != StorageMode.inMemory) { - updates = BroadcastUpdates(path); + broadcastUpdates = BroadcastUpdates(path); } - return WebDatabase(connection.database, options.mutex ?? mutex, - broadcastUpdates: updates, - profileQueries: sqliteOptions.profileQueries); + return WebDatabase( + connection.database, + options.mutex ?? mutex, + broadcastUpdates: broadcastUpdates, + profileQueries: sqliteOptions.profileQueries, + updates: updatesFor(connection.database), + ); } @override diff --git a/packages/sqlite_async/lib/src/web/worker/throttled_common_database.dart b/packages/sqlite_async/lib/src/web/worker/throttled_common_database.dart deleted file mode 100644 index 8a0e901..0000000 --- a/packages/sqlite_async/lib/src/web/worker/throttled_common_database.dart +++ /dev/null @@ -1,191 +0,0 @@ -import 'dart:async'; - -import 'package:sqlite_async/sqlite3_wasm.dart'; - -/// Wrap a CommonDatabase to throttle its updates stream. -/// This is so that we can throttle the updates _within_ -/// the worker process, avoiding mass notifications over -/// the MessagePort. -class ThrottledCommonDatabase extends CommonDatabase { - final CommonDatabase _db; - final StreamController _transactionController = - StreamController.broadcast(); - - ThrottledCommonDatabase(this._db); - - @override - int get userVersion => _db.userVersion; - - @override - set userVersion(int userVersion) { - _db.userVersion = userVersion; - } - - @override - bool get autocommit => _db.autocommit; - - @override - DatabaseConfig get config => _db.config; - - @override - void createAggregateFunction({ - required String functionName, - required AggregateFunction function, - AllowedArgumentCount argumentCount = const AllowedArgumentCount.any(), - bool deterministic = false, - bool directOnly = true, - bool subtype = false, - }) { - _db.createAggregateFunction(functionName: functionName, function: function); - } - - @override - void createCollation( - {required String name, required CollatingFunction function}) { - _db.createCollation(name: name, function: function); - } - - @override - void createFunction({ - required String functionName, - required ScalarFunction function, - AllowedArgumentCount argumentCount = const AllowedArgumentCount.any(), - bool deterministic = false, - bool directOnly = true, - bool subtype = false, - }) { - _db.createFunction(functionName: functionName, function: function); - } - - @override - void dispose() { - _db.dispose(); - } - - @override - void execute(String sql, [List parameters = const []]) { - _db.execute(sql, parameters); - } - - @override - int getUpdatedRows() { - // ignore: deprecated_member_use - return _db.getUpdatedRows(); - } - - @override - int get lastInsertRowId => _db.lastInsertRowId; - - @override - CommonPreparedStatement prepare(String sql, - {bool persistent = false, bool vtab = true, bool checkNoTail = false}) { - return _db.prepare(sql, - persistent: persistent, vtab: vtab, checkNoTail: checkNoTail); - } - - @override - List prepareMultiple(String sql, - {bool persistent = false, bool vtab = true}) { - return _db.prepareMultiple(sql, persistent: persistent, vtab: vtab); - } - - @override - ResultSet select(String sql, [List parameters = const []]) { - bool preAutocommit = _db.autocommit; - final result = _db.select(sql, parameters); - bool postAutocommit = _db.autocommit; - if (!preAutocommit && postAutocommit) { - _transactionController.add(true); - } - return result; - } - - @override - int get updatedRows => _db.updatedRows; - - @override - Stream get updates { - return throttledUpdates(_db, _transactionController.stream); - } - - @override - VoidPredicate? get commitFilter => _db.commitFilter; - - @override - set commitFilter(VoidPredicate? filter) => _db.commitFilter = filter; - - @override - Stream get commits => _db.commits; - - @override - Stream get rollbacks => _db.rollbacks; -} - -/// This throttles the database update stream to: -/// 1. Trigger max once every 1ms. -/// 2. Only trigger _after_ transactions. -Stream throttledUpdates( - CommonDatabase source, Stream transactionStream) { - StreamController? controller; - Set pendingUpdates = {}; - var paused = false; - - Timer? updateDebouncer; - - void maybeFireUpdates() { - updateDebouncer?.cancel(); - updateDebouncer = null; - - if (paused) { - // Continue collecting updates, but don't fire any - return; - } - - if (!source.autocommit) { - // Inside a transaction - do not fire updates - return; - } - - if (pendingUpdates.isNotEmpty) { - for (var update in pendingUpdates) { - controller!.add(update); - } - - pendingUpdates.clear(); - } - } - - void collectUpdate(SqliteUpdate event) { - // We merge updates with the same kind and tableName. - // rowId is never used in sqlite_async. - pendingUpdates.add(SqliteUpdate(event.kind, event.tableName, 0)); - - updateDebouncer ??= - Timer(const Duration(milliseconds: 1), maybeFireUpdates); - } - - StreamSubscription? txSubscription; - StreamSubscription? sourceSubscription; - - controller = StreamController(onListen: () { - txSubscription = transactionStream.listen((event) { - maybeFireUpdates(); - }, onError: (error) { - controller?.addError(error); - }); - - sourceSubscription = source.updates.listen(collectUpdate, onError: (error) { - controller?.addError(error); - }); - }, onPause: () { - paused = true; - }, onResume: () { - paused = false; - maybeFireUpdates(); - }, onCancel: () { - txSubscription?.cancel(); - sourceSubscription?.cancel(); - }); - - return controller.stream; -} diff --git a/packages/sqlite_async/lib/src/web/worker/worker_utils.dart b/packages/sqlite_async/lib/src/web/worker/worker_utils.dart index 3ecb257..7603306 100644 --- a/packages/sqlite_async/lib/src/web/worker/worker_utils.dart +++ b/packages/sqlite_async/lib/src/web/worker/worker_utils.dart @@ -1,3 +1,4 @@ +import 'dart:async'; import 'dart:js_interop'; import 'dart:js_interop_unsafe'; @@ -6,8 +7,7 @@ import 'package:mutex/mutex.dart'; import 'package:sqlite3/wasm.dart'; import 'package:sqlite3_web/sqlite3_web.dart'; import 'package:sqlite3_web/protocol_utils.dart' as proto; - -import 'throttled_common_database.dart'; +import 'package:sqlite_async/src/utils/database_utils.dart'; import '../protocol.dart'; @@ -22,13 +22,9 @@ base class AsyncSqliteController extends DatabaseController { // Register any custom functions here if needed - final throttled = ThrottledCommonDatabase(db); - - return AsyncSqliteDatabase(database: throttled); + return AsyncSqliteDatabase(database: db); } - /// Opens a database with the `sqlite3` package that will be wrapped in a - /// [ThrottledCommonDatabase] for [openDatabase]. @visibleForOverriding CommonDatabase openUnderlying( WasmSqlite3 sqlite3, @@ -51,6 +47,7 @@ base class AsyncSqliteController extends DatabaseController { class AsyncSqliteDatabase extends WorkerDatabase { @override final CommonDatabase database; + final Stream> _updates; // This mutex is only used for lock requests from clients. Clients only send // these requests for shared workers, so we can assume each database is only @@ -58,7 +55,8 @@ class AsyncSqliteDatabase extends WorkerDatabase { final mutex = ReadWriteMutex(); final Map _state = {}; - AsyncSqliteDatabase({required this.database}); + AsyncSqliteDatabase({required this.database}) + : _updates = database.throttledUpdatedTables; _ConnectionState _findState(ClientConnection connection) { return _state.putIfAbsent(connection, _ConnectionState.new); @@ -67,9 +65,15 @@ class AsyncSqliteDatabase extends WorkerDatabase { void _markHoldsMutex(ClientConnection connection) { final state = _findState(connection); state.holdsMutex = true; + _registerCloseListener(state, connection); + } + + void _registerCloseListener( + _ConnectionState state, ClientConnection connection) { if (!state.hasOnCloseListener) { state.hasOnCloseListener = true; connection.closed.then((_) { + state.unsubscribeUpdates(); if (state.holdsMutex) { mutex.release(); } @@ -93,6 +97,7 @@ class AsyncSqliteDatabase extends WorkerDatabase { _findState(connection).holdsMutex = false; mutex.release(); case CustomDatabaseMessageKind.lockObtained: + case CustomDatabaseMessageKind.notifyUpdates: throw UnsupportedError('This is a response, not a request'); case CustomDatabaseMessageKind.getAutoCommit: return database.autocommit.toJS; @@ -129,6 +134,25 @@ class AsyncSqliteDatabase extends WorkerDatabase { "Transaction rolled back by earlier statement. Cannot execute: $sql"); } database.execute(sql, parameters); + case CustomDatabaseMessageKind.updateSubscriptionManagement: + final shouldSubscribe = (message.rawParameters[0] as JSBoolean).toDart; + final id = message.rawSql.toDart; + final state = _findState(connection); + + if (shouldSubscribe) { + state.unsubscribeUpdates(); + _registerCloseListener(state, connection); + + state.updatesNotification = _updates.listen((tables) { + connection.customRequest(CustomDatabaseMessage( + CustomDatabaseMessageKind.notifyUpdates, + id, + tables.toList(), + )); + }); + } else { + state.unsubscribeUpdates(); + } } return CustomDatabaseMessage(CustomDatabaseMessageKind.lockObtained); @@ -148,4 +172,12 @@ class AsyncSqliteDatabase extends WorkerDatabase { final class _ConnectionState { bool hasOnCloseListener = false; bool holdsMutex = false; + StreamSubscription>? updatesNotification; + + void unsubscribeUpdates() { + if (updatesNotification case final active?) { + updatesNotification = null; + active.cancel(); + } + } } diff --git a/packages/sqlite_async/lib/web.dart b/packages/sqlite_async/lib/web.dart index 3a65115..f151b81 100644 --- a/packages/sqlite_async/lib/web.dart +++ b/packages/sqlite_async/lib/web.dart @@ -4,12 +4,15 @@ /// workers. library; +import 'dart:js_interop'; + import 'package:sqlite3_web/sqlite3_web.dart'; import 'package:web/web.dart'; import 'sqlite3_common.dart'; import 'sqlite_async.dart'; import 'src/web/database.dart'; +import 'src/web/update_notifications.dart'; /// An endpoint that can be used, by any running JavaScript context in the same /// website, to connect to an existing [WebSqliteConnection]. @@ -33,6 +36,13 @@ typedef WebDatabaseEndpoint = ({ /// compiling for the web. abstract mixin class WebSqliteOpenFactory implements SqliteOpenFactory { + final UpdateNotificationStreams _updateStreams = UpdateNotificationStreams(); + + /// Handles a custom request sent from the worker to the client. + Future handleCustomRequest(JSAny? request) { + return _updateStreams.handleRequest(request); + } + /// Opens a [WebSqlite] instance for the given [options]. /// /// This method can be overriden in scenarios where the way [WebSqlite] is @@ -43,6 +53,7 @@ abstract mixin class WebSqliteOpenFactory return WebSqlite.open( worker: Uri.parse(options.workerUri), wasmModule: Uri.parse(options.wasmUri), + handleCustomRequest: handleCustomRequest, ); } @@ -54,6 +65,14 @@ abstract mixin class WebSqliteOpenFactory WebSqlite sqlite, String name) { return sqlite.connectToRecommended(name); } + + /// Obtains a stream of [UpdateNotification]s from a [database]. + /// + /// The default implementation uses custom requests to allow workers to + /// debounce the stream on their side to avoid messages where possible. + Stream updatesFor(Database database) { + return _updateStreams.updatesFor(database); + } } /// A [SqliteConnection] interface implemented by opened connections when @@ -85,8 +104,11 @@ abstract class WebSqliteConnection implements SqliteConnection { /// contexts to exchange opened database connections. static Future connectToEndpoint( WebDatabaseEndpoint endpoint) async { + final updates = UpdateNotificationStreams(); final rawSqlite = await WebSqlite.connectToPort( - (endpoint.connectPort, endpoint.connectName)); + (endpoint.connectPort, endpoint.connectName), + handleCustomRequest: updates.handleRequest, + ); final database = WebDatabase( rawSqlite, @@ -95,6 +117,7 @@ abstract class WebSqliteConnection implements SqliteConnection { null => null, }, profileQueries: false, + updates: updates.updatesFor(rawSqlite), ); return database; } From 64260569f6491a137dd680d7df62938befef48e0 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Thu, 7 Aug 2025 12:02:24 +0200 Subject: [PATCH 74/90] Fix tests --- .../native_sqlite_connection_impl.dart | 43 +++---------------- .../lib/src/utils/shared_utils.dart | 1 + .../lib/src/web/update_notifications.dart | 5 ++- .../lib/src/web/web_sqlite_open_factory.dart | 11 ++--- packages/sqlite_async/lib/web.dart | 4 +- scripts/sqlite3_wasm_download.dart | 2 +- 6 files changed, 19 insertions(+), 47 deletions(-) diff --git a/packages/sqlite_async/lib/src/native/database/native_sqlite_connection_impl.dart b/packages/sqlite_async/lib/src/native/database/native_sqlite_connection_impl.dart index e2df0f3..301d0af 100644 --- a/packages/sqlite_async/lib/src/native/database/native_sqlite_connection_impl.dart +++ b/packages/sqlite_async/lib/src/native/database/native_sqlite_connection_impl.dart @@ -303,40 +303,13 @@ Future _sqliteConnectionIsolateInner(_SqliteConnectionParams params, final server = params.portServer; final commandPort = ReceivePort(); - Timer? updateDebouncer; - Set updatedTables = {}; + db.throttledUpdatedTables.listen((changedTables) { + client.fire(UpdateNotification(changedTables)); + }); + int? txId; Object? txError; - void maybeFireUpdates() { - // We keep buffering the set of updated tables until we are not - // in a transaction. Firing transactions inside a transaction - // has multiple issues: - // 1. Watched queries would detect changes to the underlying tables, - // but the data would not be visible to queries yet. - // 2. It would trigger many more notifications than required. - // - // This still includes updates for transactions that are rolled back. - // We could handle those better at a later stage. - - if (updatedTables.isNotEmpty && db.autocommit) { - client.fire(UpdateNotification(updatedTables)); - updatedTables.clear(); - } - updateDebouncer?.cancel(); - updateDebouncer = null; - } - - db.updates.listen((event) { - updatedTables.add(event.tableName); - - // This handles two cases: - // 1. Update arrived after _SqliteIsolateClose (not sure if this could happen). - // 2. Long-running _SqliteIsolateClosure that should fire updates while running. - updateDebouncer ??= - Timer(const Duration(milliseconds: 1), maybeFireUpdates); - }); - ResultSet runStatement(_SqliteIsolateStatement data) { if (data.sql == 'BEGIN' || data.sql == 'BEGIN IMMEDIATE') { if (txId != null) { @@ -388,8 +361,6 @@ Future _sqliteConnectionIsolateInner(_SqliteConnectionParams params, throw sqlite.SqliteException( 0, 'Transaction must be closed within the read or write lock'); } - // We would likely have received updates by this point - fire now. - maybeFireUpdates(); return null; case _SqliteIsolateStatement(): return task.timeSync( @@ -399,11 +370,7 @@ Future _sqliteConnectionIsolateInner(_SqliteConnectionParams params, parameters: data.args, ); case _SqliteIsolateClosure(): - try { - return await data.cb(db); - } finally { - maybeFireUpdates(); - } + return await data.cb(db); case _SqliteIsolateConnectionClose(): db.dispose(); return null; diff --git a/packages/sqlite_async/lib/src/utils/shared_utils.dart b/packages/sqlite_async/lib/src/utils/shared_utils.dart index ff291f4..0e9c9d6 100644 --- a/packages/sqlite_async/lib/src/utils/shared_utils.dart +++ b/packages/sqlite_async/lib/src/utils/shared_utils.dart @@ -123,6 +123,7 @@ extension ThrottledUpdates on CommonDatabase { controller = StreamController(onListen: () { txSubscription = commits.listen((_) { + print('did commit'); maybeFireUpdates(); }, onError: (error) { controller?.addError(error); diff --git a/packages/sqlite_async/lib/src/web/update_notifications.dart b/packages/sqlite_async/lib/src/web/update_notifications.dart index ecea60d..f04a785 100644 --- a/packages/sqlite_async/lib/src/web/update_notifications.dart +++ b/packages/sqlite_async/lib/src/web/update_notifications.dart @@ -26,7 +26,8 @@ final class UpdateNotificationStreams { .map((e) => (e as JSString).toDart) .toSet()); - _updates[customRequest.rawSql.toDart]?.add(notification); + final controller = _updates[customRequest.rawSql.toDart]; + controller?.add(notification); } return null; @@ -50,6 +51,8 @@ final class UpdateNotificationStreams { id, [false], )); + + _updates.remove(id); }; return controller.stream; diff --git a/packages/sqlite_async/lib/src/web/web_sqlite_open_factory.dart b/packages/sqlite_async/lib/src/web/web_sqlite_open_factory.dart index 205734f..513b1f2 100644 --- a/packages/sqlite_async/lib/src/web/web_sqlite_open_factory.dart +++ b/packages/sqlite_async/lib/src/web/web_sqlite_open_factory.dart @@ -10,7 +10,7 @@ import 'package:sqlite_async/web.dart'; import 'database.dart'; import 'worker/worker_utils.dart'; -Map> webSQLiteImplementations = {}; +Map> _webSQLiteImplementations = {}; /// Web implementation of [AbstractDefaultSqliteOpenFactory] class DefaultSqliteOpenFactory @@ -20,13 +20,13 @@ class DefaultSqliteOpenFactory final cacheKey = sqliteOptions.webSqliteOptions.wasmUri + sqliteOptions.webSqliteOptions.workerUri; - if (webSQLiteImplementations.containsKey(cacheKey)) { - return webSQLiteImplementations[cacheKey]!; + if (_webSQLiteImplementations.containsKey(cacheKey)) { + return _webSQLiteImplementations[cacheKey]!; } - webSQLiteImplementations[cacheKey] = + _webSQLiteImplementations[cacheKey] = openWebSqlite(sqliteOptions.webSqliteOptions); - return webSQLiteImplementations[cacheKey]!; + return _webSQLiteImplementations[cacheKey]!; }); DefaultSqliteOpenFactory( @@ -42,6 +42,7 @@ class DefaultSqliteOpenFactory wasmModule: Uri.parse(sqliteOptions.webSqliteOptions.wasmUri), worker: Uri.parse(sqliteOptions.webSqliteOptions.workerUri), controller: AsyncSqliteController(), + handleCustomRequest: handleCustomRequest, ); } diff --git a/packages/sqlite_async/lib/web.dart b/packages/sqlite_async/lib/web.dart index f151b81..e318697 100644 --- a/packages/sqlite_async/lib/web.dart +++ b/packages/sqlite_async/lib/web.dart @@ -29,6 +29,8 @@ typedef WebDatabaseEndpoint = ({ String? lockName, }); +final UpdateNotificationStreams _updateStreams = UpdateNotificationStreams(); + /// An additional interface for [SqliteOpenFactory] exposing additional /// functionality that is only relevant when compiling to the web. /// @@ -36,8 +38,6 @@ typedef WebDatabaseEndpoint = ({ /// compiling for the web. abstract mixin class WebSqliteOpenFactory implements SqliteOpenFactory { - final UpdateNotificationStreams _updateStreams = UpdateNotificationStreams(); - /// Handles a custom request sent from the worker to the client. Future handleCustomRequest(JSAny? request) { return _updateStreams.handleRequest(request); diff --git a/scripts/sqlite3_wasm_download.dart b/scripts/sqlite3_wasm_download.dart index 62acbbe..9716eee 100644 --- a/scripts/sqlite3_wasm_download.dart +++ b/scripts/sqlite3_wasm_download.dart @@ -4,7 +4,7 @@ library; import 'dart:io'; final sqliteUrl = - 'https://github.com/simolus3/sqlite3.dart/releases/download/sqlite3-2.4.3/sqlite3.wasm'; + 'https://github.com/simolus3/sqlite3.dart/releases/download/sqlite3-2.8.0/sqlite3.wasm'; void main() async { // Create assets directory if it doesn't exist From bc66337140941e8e57dab988a2339b70c8a02273 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Thu, 7 Aug 2025 12:18:58 +0200 Subject: [PATCH 75/90] Remove debug print --- packages/sqlite_async/lib/src/utils/shared_utils.dart | 1 - packages/sqlite_async/pubspec.yaml | 6 +++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/packages/sqlite_async/lib/src/utils/shared_utils.dart b/packages/sqlite_async/lib/src/utils/shared_utils.dart index 0e9c9d6..ff291f4 100644 --- a/packages/sqlite_async/lib/src/utils/shared_utils.dart +++ b/packages/sqlite_async/lib/src/utils/shared_utils.dart @@ -123,7 +123,6 @@ extension ThrottledUpdates on CommonDatabase { controller = StreamController(onListen: () { txSubscription = commits.listen((_) { - print('did commit'); maybeFireUpdates(); }, onError: (error) { controller?.addError(error); diff --git a/packages/sqlite_async/pubspec.yaml b/packages/sqlite_async/pubspec.yaml index a155f2b..7d7c1c3 100644 --- a/packages/sqlite_async/pubspec.yaml +++ b/packages/sqlite_async/pubspec.yaml @@ -3,7 +3,7 @@ description: High-performance asynchronous interface for SQLite on Dart and Flut version: 0.11.8 repository: https://github.com/powersync-ja/sqlite_async.dart environment: - sdk: ">=3.5.0 <4.0.0" + sdk: ">=3.6.0 <4.0.0" topics: - sqlite @@ -12,8 +12,8 @@ topics: - flutter dependencies: - sqlite3: ^2.8.0 - sqlite3_web: ^0.3.0 + sqlite3: ^2.9.0 + sqlite3_web: ^0.3.1 async: ^2.10.0 collection: ^1.17.0 mutex: ^3.1.0 From ed46766541cc90a5c2f459772665deacba7f988b Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Thu, 7 Aug 2025 14:39:11 +0200 Subject: [PATCH 76/90] Update sqlite3_web as well --- packages/sqlite_async/pubspec.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/sqlite_async/pubspec.yaml b/packages/sqlite_async/pubspec.yaml index 7d7c1c3..586bcc6 100644 --- a/packages/sqlite_async/pubspec.yaml +++ b/packages/sqlite_async/pubspec.yaml @@ -13,7 +13,7 @@ topics: dependencies: sqlite3: ^2.9.0 - sqlite3_web: ^0.3.1 + sqlite3_web: ^0.3.2 async: ^2.10.0 collection: ^1.17.0 mutex: ^3.1.0 From e0a1a3c0424892aeed965814d4538ac75fc07de0 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Thu, 7 Aug 2025 15:44:28 +0200 Subject: [PATCH 77/90] Fix import --- packages/sqlite_async/lib/src/web/worker/worker_utils.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/sqlite_async/lib/src/web/worker/worker_utils.dart b/packages/sqlite_async/lib/src/web/worker/worker_utils.dart index 7603306..265f4f7 100644 --- a/packages/sqlite_async/lib/src/web/worker/worker_utils.dart +++ b/packages/sqlite_async/lib/src/web/worker/worker_utils.dart @@ -7,7 +7,7 @@ import 'package:mutex/mutex.dart'; import 'package:sqlite3/wasm.dart'; import 'package:sqlite3_web/sqlite3_web.dart'; import 'package:sqlite3_web/protocol_utils.dart' as proto; -import 'package:sqlite_async/src/utils/database_utils.dart'; +import 'package:sqlite_async/src/utils/shared_utils.dart'; import '../protocol.dart'; From 6f4c6a065c8da6805e633d92fa9d75cbb09bb0a1 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Fri, 8 Aug 2025 10:49:36 +0200 Subject: [PATCH 78/90] Avoid updating minimum SDK constraint --- packages/sqlite_async/lib/src/web/worker/worker_utils.dart | 3 ++- packages/sqlite_async/pubspec.yaml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/sqlite_async/lib/src/web/worker/worker_utils.dart b/packages/sqlite_async/lib/src/web/worker/worker_utils.dart index 265f4f7..0c3e8f7 100644 --- a/packages/sqlite_async/lib/src/web/worker/worker_utils.dart +++ b/packages/sqlite_async/lib/src/web/worker/worker_utils.dart @@ -135,7 +135,8 @@ class AsyncSqliteDatabase extends WorkerDatabase { } database.execute(sql, parameters); case CustomDatabaseMessageKind.updateSubscriptionManagement: - final shouldSubscribe = (message.rawParameters[0] as JSBoolean).toDart; + final shouldSubscribe = + (message.rawParameters.toDart[0] as JSBoolean).toDart; final id = message.rawSql.toDart; final state = _findState(connection); diff --git a/packages/sqlite_async/pubspec.yaml b/packages/sqlite_async/pubspec.yaml index 586bcc6..464bd2d 100644 --- a/packages/sqlite_async/pubspec.yaml +++ b/packages/sqlite_async/pubspec.yaml @@ -3,7 +3,7 @@ description: High-performance asynchronous interface for SQLite on Dart and Flut version: 0.11.8 repository: https://github.com/powersync-ja/sqlite_async.dart environment: - sdk: ">=3.6.0 <4.0.0" + sdk: ">=3.5.0 <4.0.0" topics: - sqlite From eaab81ea6aacf8e8ac3da08f1d32cb6db91bdf6a Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Fri, 8 Aug 2025 11:18:17 +0200 Subject: [PATCH 79/90] chore(release): publish packages - sqlite_async@0.12.0 - drift_sqlite_async@0.2.3+1 --- CHANGELOG.md | 28 ++++++++++++++++++++++++ packages/drift_sqlite_async/CHANGELOG.md | 4 ++++ packages/drift_sqlite_async/pubspec.yaml | 4 ++-- packages/sqlite_async/CHANGELOG.md | 4 ++++ packages/sqlite_async/pubspec.yaml | 2 +- 5 files changed, 39 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cafde71..93a8b0d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,34 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## 2025-08-08 + +### Changes + +--- + +Packages with breaking changes: + + - There are no breaking changes in this release. + +Packages with other changes: + + - [`sqlite_async` - `v0.12.0`](#sqlite_async---v0120) + - [`drift_sqlite_async` - `v0.2.3+1`](#drift_sqlite_async---v0231) + +Packages with dependency updates only: + +> Packages listed below depend on other packages in this workspace that have had changes. Their versions have been incremented to bump the minimum dependency versions of the packages they depend upon in this project. + + - `drift_sqlite_async` - `v0.2.3+1` + +--- + +#### `sqlite_async` - `v0.12.0` + + - Avoid large transactions creating a large internal update queue. + + ## 2025-07-29 --- diff --git a/packages/drift_sqlite_async/CHANGELOG.md b/packages/drift_sqlite_async/CHANGELOG.md index e70cd40..7c381da 100644 --- a/packages/drift_sqlite_async/CHANGELOG.md +++ b/packages/drift_sqlite_async/CHANGELOG.md @@ -1,3 +1,7 @@ +## 0.2.3+1 + + - Update a dependency to the latest release. + ## 0.2.3 - Support nested transactions. diff --git a/packages/drift_sqlite_async/pubspec.yaml b/packages/drift_sqlite_async/pubspec.yaml index 77f250d..b865809 100644 --- a/packages/drift_sqlite_async/pubspec.yaml +++ b/packages/drift_sqlite_async/pubspec.yaml @@ -1,5 +1,5 @@ name: drift_sqlite_async -version: 0.2.3 +version: 0.2.3+1 homepage: https://github.com/powersync-ja/sqlite_async.dart repository: https://github.com/powersync-ja/sqlite_async.dart description: Use Drift with a sqlite_async database, allowing both to be used in the same application. @@ -15,7 +15,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: drift: ">=2.28.0 <3.0.0" - sqlite_async: ^0.11.8 + sqlite_async: ^0.12.0 dev_dependencies: build_runner: ^2.4.8 diff --git a/packages/sqlite_async/CHANGELOG.md b/packages/sqlite_async/CHANGELOG.md index 387c944..8cda2a6 100644 --- a/packages/sqlite_async/CHANGELOG.md +++ b/packages/sqlite_async/CHANGELOG.md @@ -1,3 +1,7 @@ +## 0.12.0 + + - Avoid large transactions creating a large internal update queue. + ## 0.11.8 - Support nested transactions (emulated with `SAVEPOINT` statements). diff --git a/packages/sqlite_async/pubspec.yaml b/packages/sqlite_async/pubspec.yaml index 464bd2d..6e62e52 100644 --- a/packages/sqlite_async/pubspec.yaml +++ b/packages/sqlite_async/pubspec.yaml @@ -1,6 +1,6 @@ name: sqlite_async description: High-performance asynchronous interface for SQLite on Dart and Flutter. -version: 0.11.8 +version: 0.12.0 repository: https://github.com/powersync-ja/sqlite_async.dart environment: sdk: ">=3.5.0 <4.0.0" From 0e2fa41567e96f181cd43d5bb25a3321c941c78f Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Fri, 8 Aug 2025 16:39:36 +0200 Subject: [PATCH 80/90] Make table updates a multi-subscription stream --- .../native_sqlite_connection_impl.dart | 2 +- .../lib/src/utils/shared_utils.dart | 123 ++++++++++-------- .../lib/src/web/worker/worker_utils.dart | 3 +- 3 files changed, 74 insertions(+), 54 deletions(-) diff --git a/packages/sqlite_async/lib/src/native/database/native_sqlite_connection_impl.dart b/packages/sqlite_async/lib/src/native/database/native_sqlite_connection_impl.dart index 301d0af..e90739e 100644 --- a/packages/sqlite_async/lib/src/native/database/native_sqlite_connection_impl.dart +++ b/packages/sqlite_async/lib/src/native/database/native_sqlite_connection_impl.dart @@ -303,7 +303,7 @@ Future _sqliteConnectionIsolateInner(_SqliteConnectionParams params, final server = params.portServer; final commandPort = ReceivePort(); - db.throttledUpdatedTables.listen((changedTables) { + db.updatedTables.listen((changedTables) { client.fire(UpdateNotification(changedTables)); }); diff --git a/packages/sqlite_async/lib/src/utils/shared_utils.dart b/packages/sqlite_async/lib/src/utils/shared_utils.dart index ff291f4..ad9a2f0 100644 --- a/packages/sqlite_async/lib/src/utils/shared_utils.dart +++ b/packages/sqlite_async/lib/src/utils/shared_utils.dart @@ -79,68 +79,87 @@ List mapParameters(List parameters) { } extension ThrottledUpdates on CommonDatabase { - /// Wraps [updatesSync] to: + /// An unthrottled stream of updated tables that emits on every commit. /// - /// - Not fire in transactions. - /// - Fire asynchronously. - /// - Only report table names, which are buffered to avoid duplicates. - Stream> get throttledUpdatedTables { - StreamController>? controller; - var pendingUpdates = {}; - var paused = false; - - Timer? updateDebouncer; - - void maybeFireUpdates() { - updateDebouncer?.cancel(); - updateDebouncer = null; - - if (paused) { - // Continue collecting updates, but don't fire any - return; + /// A paused subscription on this stream will buffer changed tables into a + /// growing set instead of losing events, so this stream is simple to throttle + /// downstream. + Stream> get updatedTables { + final listeners = <_UpdateListener>[]; + var uncommitedUpdates = {}; + var underlyingSubscriptions = >[]; + + void handleUpdate(SqliteUpdate update) { + uncommitedUpdates.add(update.tableName); + } + + void afterCommit() { + for (final listener in listeners) { + listener.notify(uncommitedUpdates); } - if (!autocommit) { - // Inside a transaction - do not fire updates - return; + uncommitedUpdates.clear(); + } + + void afterRollback() { + uncommitedUpdates.clear(); + } + + void addListener(_UpdateListener listener) { + listeners.add(listener); + + if (listeners.length == 1) { + // First listener, start listening for raw updates on underlying + // database. + underlyingSubscriptions = [ + updatesSync.listen(handleUpdate), + commits.listen((_) => afterCommit()), + commits.listen((_) => afterRollback()) + ]; } + } - if (pendingUpdates.isNotEmpty) { - controller!.add(pendingUpdates); - pendingUpdates = {}; + void removeListener(_UpdateListener listener) { + listeners.remove(listener); + if (listeners.isEmpty) { + for (final sub in underlyingSubscriptions) { + sub.cancel(); + } } } - void collectUpdate(SqliteUpdate event) { - pendingUpdates.add(event.tableName); + return Stream.multi( + (listener) { + final wrapped = _UpdateListener(listener); + addListener(wrapped); - updateDebouncer ??= - Timer(const Duration(milliseconds: 1), maybeFireUpdates); + listener.onCancel = () => removeListener(wrapped); + }, + isBroadcast: true, + ); + } +} + +class _UpdateListener { + final MultiStreamController> downstream; + Set buffered = {}; + + _UpdateListener(this.downstream); + + void notify(Set pendingUpdates) { + buffered.addAll(pendingUpdates); + if (!downstream.isPaused) { + downstream.add(buffered); + buffered = {}; } + } +} - StreamSubscription? txSubscription; - StreamSubscription? sourceSubscription; - - controller = StreamController(onListen: () { - txSubscription = commits.listen((_) { - maybeFireUpdates(); - }, onError: (error) { - controller?.addError(error); - }); - - sourceSubscription = updatesSync.listen(collectUpdate, onError: (error) { - controller?.addError(error); - }); - }, onPause: () { - paused = true; - }, onResume: () { - paused = false; - maybeFireUpdates(); - }, onCancel: () { - txSubscription?.cancel(); - sourceSubscription?.cancel(); - }); - - return controller.stream; +extension StreamUtils on Stream { + Stream pauseAfterEvent(Duration duration) async* { + await for (final event in this) { + yield event; + await Future.delayed(duration); + } } } diff --git a/packages/sqlite_async/lib/src/web/worker/worker_utils.dart b/packages/sqlite_async/lib/src/web/worker/worker_utils.dart index 0c3e8f7..a6dae4c 100644 --- a/packages/sqlite_async/lib/src/web/worker/worker_utils.dart +++ b/packages/sqlite_async/lib/src/web/worker/worker_utils.dart @@ -56,7 +56,8 @@ class AsyncSqliteDatabase extends WorkerDatabase { final Map _state = {}; AsyncSqliteDatabase({required this.database}) - : _updates = database.throttledUpdatedTables; + : _updates = database.updatedTables + .pauseAfterEvent(const Duration(milliseconds: 1)); _ConnectionState _findState(ClientConnection connection) { return _state.putIfAbsent(connection, _ConnectionState.new); From e0938c4ef0138ee7b77b5911f54fa44fe8722f58 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Fri, 8 Aug 2025 16:45:53 +0200 Subject: [PATCH 81/90] Support multiple listeners for table updates --- packages/sqlite_async/lib/src/utils/shared_utils.dart | 9 --------- .../sqlite_async/lib/src/web/worker/worker_utils.dart | 10 +++++----- 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/packages/sqlite_async/lib/src/utils/shared_utils.dart b/packages/sqlite_async/lib/src/utils/shared_utils.dart index ad9a2f0..ae38e85 100644 --- a/packages/sqlite_async/lib/src/utils/shared_utils.dart +++ b/packages/sqlite_async/lib/src/utils/shared_utils.dart @@ -154,12 +154,3 @@ class _UpdateListener { } } } - -extension StreamUtils on Stream { - Stream pauseAfterEvent(Duration duration) async* { - await for (final event in this) { - yield event; - await Future.delayed(duration); - } - } -} diff --git a/packages/sqlite_async/lib/src/web/worker/worker_utils.dart b/packages/sqlite_async/lib/src/web/worker/worker_utils.dart index a6dae4c..059c281 100644 --- a/packages/sqlite_async/lib/src/web/worker/worker_utils.dart +++ b/packages/sqlite_async/lib/src/web/worker/worker_utils.dart @@ -56,8 +56,7 @@ class AsyncSqliteDatabase extends WorkerDatabase { final Map _state = {}; AsyncSqliteDatabase({required this.database}) - : _updates = database.updatedTables - .pauseAfterEvent(const Duration(milliseconds: 1)); + : _updates = database.updatedTables; _ConnectionState _findState(ClientConnection connection) { return _state.putIfAbsent(connection, _ConnectionState.new); @@ -145,12 +144,13 @@ class AsyncSqliteDatabase extends WorkerDatabase { state.unsubscribeUpdates(); _registerCloseListener(state, connection); - state.updatesNotification = _updates.listen((tables) { - connection.customRequest(CustomDatabaseMessage( + late StreamSubscription subscription; + subscription = state.updatesNotification = _updates.listen((tables) { + subscription.pause(connection.customRequest(CustomDatabaseMessage( CustomDatabaseMessageKind.notifyUpdates, id, tables.toList(), - )); + ))); }); } else { state.unsubscribeUpdates(); From b9c8d2d6a9879eda6e7293ab967c529f1cb9e9af Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Fri, 8 Aug 2025 16:48:40 +0200 Subject: [PATCH 82/90] Prepare release --- CHANGELOG.md | 5 +++++ packages/sqlite_async/CHANGELOG.md | 4 ++++ packages/sqlite_async/pubspec.yaml | 2 +- 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 93a8b0d..821f8db 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ Packages with breaking changes: Packages with other changes: + - [`sqlite_async` - `v0.12.1`](#sqlite_async---v0121) - [`sqlite_async` - `v0.12.0`](#sqlite_async---v0120) - [`drift_sqlite_async` - `v0.2.3+1`](#drift_sqlite_async---v0231) @@ -26,6 +27,10 @@ Packages with dependency updates only: --- +#### `sqlite_async` - `v0.12.1` + +- Fix distributing updates from shared worker. + #### `sqlite_async` - `v0.12.0` - Avoid large transactions creating a large internal update queue. diff --git a/packages/sqlite_async/CHANGELOG.md b/packages/sqlite_async/CHANGELOG.md index 8cda2a6..342adca 100644 --- a/packages/sqlite_async/CHANGELOG.md +++ b/packages/sqlite_async/CHANGELOG.md @@ -1,3 +1,7 @@ +## 0.12.1 + +- Fix distributing updates from shared worker. + ## 0.12.0 - Avoid large transactions creating a large internal update queue. diff --git a/packages/sqlite_async/pubspec.yaml b/packages/sqlite_async/pubspec.yaml index 6e62e52..4b45d57 100644 --- a/packages/sqlite_async/pubspec.yaml +++ b/packages/sqlite_async/pubspec.yaml @@ -1,6 +1,6 @@ name: sqlite_async description: High-performance asynchronous interface for SQLite on Dart and Flutter. -version: 0.12.0 +version: 0.12.1 repository: https://github.com/powersync-ja/sqlite_async.dart environment: sdk: ">=3.5.0 <4.0.0" From 6705d13335df92b8f548a182e53a097a91c16e37 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Fri, 8 Aug 2025 16:57:24 +0200 Subject: [PATCH 83/90] Add tests --- .../lib/src/utils/shared_utils.dart | 7 +++ .../sqlite_async/test/native/watch_test.dart | 46 +++++++++++++++++++ 2 files changed, 53 insertions(+) diff --git a/packages/sqlite_async/lib/src/utils/shared_utils.dart b/packages/sqlite_async/lib/src/utils/shared_utils.dart index ae38e85..542611e 100644 --- a/packages/sqlite_async/lib/src/utils/shared_utils.dart +++ b/packages/sqlite_async/lib/src/utils/shared_utils.dart @@ -133,6 +133,7 @@ extension ThrottledUpdates on CommonDatabase { final wrapped = _UpdateListener(listener); addListener(wrapped); + listener.onResume = wrapped.addPending; listener.onCancel = () => removeListener(wrapped); }, isBroadcast: true, @@ -149,6 +150,12 @@ class _UpdateListener { void notify(Set pendingUpdates) { buffered.addAll(pendingUpdates); if (!downstream.isPaused) { + addPending(); + } + } + + void addPending() { + if (buffered.isNotEmpty) { downstream.add(buffered); buffered = {}; } diff --git a/packages/sqlite_async/test/native/watch_test.dart b/packages/sqlite_async/test/native/watch_test.dart index 4e4fb83..0a97b17 100644 --- a/packages/sqlite_async/test/native/watch_test.dart +++ b/packages/sqlite_async/test/native/watch_test.dart @@ -7,6 +7,7 @@ import 'dart:math'; import 'package:sqlite3/common.dart'; import 'package:sqlite_async/sqlite_async.dart'; +import 'package:sqlite_async/src/utils/shared_utils.dart'; import 'package:test/test.dart'; import '../utils/test_utils_impl.dart'; @@ -31,6 +32,51 @@ void main() { return db; }); + test('raw update notifications', () async { + final factory = await testUtils.testFactory(path: path); + final db = factory + .openDB(SqliteOpenOptions(primaryConnection: true, readOnly: false)); + + db.execute('CREATE TABLE a (bar INTEGER);'); + db.execute('CREATE TABLE b (bar INTEGER);'); + final events = >[]; + final subscription = db.updatedTables.listen(events.add); + + db.execute('insert into a default values'); + expect(events, isEmpty); // should be async + await pumpEventQueue(); + expect(events.removeLast(), {'a'}); + + db.execute('begin'); + db.execute('insert into a default values'); + db.execute('insert into b default values'); + await pumpEventQueue(); + expect(events, isEmpty); // should only trigger on commit + db.execute('commit'); + + await pumpEventQueue(); + expect(events.removeLast(), {'a', 'b'}); + + db.execute('begin'); + db.execute('insert into a default values'); + db.execute('rollback'); + expect(events, isEmpty); + await pumpEventQueue(); + expect(events, isEmpty); // should ignore cancelled transactions + + // Should still listen during pause, and dispatch on resume + subscription.pause(); + db.execute('insert into a default values'); + await pumpEventQueue(); + expect(events, isEmpty); + + subscription.resume(); + await pumpEventQueue(); + expect(events.removeLast(), {'a'}); + + subscription.pause(); + }); + test('watch in isolate', () async { final db = await testUtils.setupDatabase(path: path); await createTables(db); From d76435962555b05f472ae0d2aa2621f0ead38004 Mon Sep 17 00:00:00 2001 From: David Martos Date: Fri, 19 Sep 2025 12:11:17 +0200 Subject: [PATCH 84/90] update doc --- packages/drift_sqlite_async/lib/src/connection.dart | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/drift_sqlite_async/lib/src/connection.dart b/packages/drift_sqlite_async/lib/src/connection.dart index fa56e2e..6c34ed4 100644 --- a/packages/drift_sqlite_async/lib/src/connection.dart +++ b/packages/drift_sqlite_async/lib/src/connection.dart @@ -15,6 +15,8 @@ import 'package:sqlite_async/sqlite_async.dart'; class SqliteAsyncDriftConnection extends DatabaseConnection { late StreamSubscription _updateSubscription; + /// [transformTableUpdates] is useful to map local table names from PowerSync that are backed by a view name + /// which is the entity that the user interacts with. SqliteAsyncDriftConnection( SqliteConnection db, { bool logStatements = false, @@ -22,8 +24,6 @@ class SqliteAsyncDriftConnection extends DatabaseConnection { }) : super(SqliteAsyncQueryExecutor(db, logStatements: logStatements)) { _updateSubscription = (db as SqliteQueries).updates!.listen((event) { final Set setUpdates; - // This is useful to map local table names from PowerSync that are backed by a view name - // which is the entity that the user interacts with. if (transformTableUpdates != null) { setUpdates = transformTableUpdates(event); } else { From 09b841ba9b34a1e309504808e399695239e08c17 Mon Sep 17 00:00:00 2001 From: David Martos Date: Fri, 19 Sep 2025 14:14:10 +0200 Subject: [PATCH 85/90] add test --- .../drift_sqlite_async/test/basic_test.dart | 40 +++++++++++++++++++ .../test/generated/database.dart | 2 + 2 files changed, 42 insertions(+) diff --git a/packages/drift_sqlite_async/test/basic_test.dart b/packages/drift_sqlite_async/test/basic_test.dart index 339cc04..dd7a159 100644 --- a/packages/drift_sqlite_async/test/basic_test.dart +++ b/packages/drift_sqlite_async/test/basic_test.dart @@ -11,6 +11,7 @@ import 'package:sqlite_async/sqlite_async.dart'; import 'package:test/test.dart'; import './utils/test_utils.dart'; +import 'generated/database.dart'; class EmptyDatabase extends GeneratedDatabase { EmptyDatabase(super.executor); @@ -245,4 +246,43 @@ INSERT INTO test_data(description) VALUES('test data'); expect(row, isEmpty); }); }); + + test('transform table updates', () async { + final path = dbPath(); + await cleanDb(path: path); + + final db = await setupDatabase(path: path); + final connection = SqliteAsyncDriftConnection( + db, + // tables with the local_ prefix are mapped to the name without the prefix + transformTableUpdates: (event) { + final updates = {}; + + for (final originalTableName in event.tables) { + final effectiveName = originalTableName.startsWith("local_") + ? originalTableName.substring(6) + : originalTableName; + updates.add(TableUpdate(effectiveName)); + } + + return updates; + }, + ); + + // Create table with a different name than drift. (Mimicking a table name backed by a view in PowerSync with the optional sync strategy) + await db.execute( + 'CREATE TABLE local_todos(id INTEGER PRIMARY KEY AUTOINCREMENT, description TEXT)', + ); + + final dbu = TodoDatabase.fromSqliteAsyncConnection(connection); + + final tableUpdatesFut = + dbu.tableUpdates(TableUpdateQuery.onTableName("todos")).first; + + // This insert will trigger the sqlite_async "updates" stream + await db.execute("INSERT INTO local_todos(description) VALUES('Test 1')"); + + expect(await tableUpdatesFut.timeout(const Duration(seconds: 2)), + {TableUpdate("todos")}); + }); } diff --git a/packages/drift_sqlite_async/test/generated/database.dart b/packages/drift_sqlite_async/test/generated/database.dart index 928c7dd..b8a5109 100644 --- a/packages/drift_sqlite_async/test/generated/database.dart +++ b/packages/drift_sqlite_async/test/generated/database.dart @@ -15,6 +15,8 @@ class TodoItems extends Table { @DriftDatabase(tables: [TodoItems]) class TodoDatabase extends _$TodoDatabase { TodoDatabase(SqliteConnection db) : super(SqliteAsyncDriftConnection(db)); + + TodoDatabase.fromSqliteAsyncConnection(SqliteAsyncDriftConnection super.conn); @override int get schemaVersion => 1; From 0a3d31e3b686ae23673f458eef1e6afb8658acf4 Mon Sep 17 00:00:00 2001 From: David Martos Date: Thu, 25 Sep 2025 19:17:54 +0200 Subject: [PATCH 86/90] Update basic_test.dart Remove legacy skip. It now supports nested transactions --- packages/drift_sqlite_async/test/basic_test.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/drift_sqlite_async/test/basic_test.dart b/packages/drift_sqlite_async/test/basic_test.dart index dd7a159..cac6f55 100644 --- a/packages/drift_sqlite_async/test/basic_test.dart +++ b/packages/drift_sqlite_async/test/basic_test.dart @@ -183,7 +183,7 @@ void main() { {'description': 'Test 1'}, {'description': 'Test 3'} ])); - }, skip: 'sqlite_async does not support nested transactions'); + }); test('Concurrent select', () async { var completer1 = Completer(); From 1252190e3522de429b751ecaadeb301137cb3d68 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Mon, 29 Sep 2025 16:07:25 +0200 Subject: [PATCH 87/90] Run CI for pull requests --- .github/workflows/test.yaml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index cb1814b..0e0d34b 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -3,13 +3,15 @@ name: Test on: push: branches: - - "**" + - "*" + pull_request: jobs: build: runs-on: ubuntu-latest + if: github.event_name == 'push' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository) steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v5 - uses: dart-lang/setup-dart@v1 - name: Install Melos @@ -29,6 +31,7 @@ jobs: test: runs-on: ubuntu-latest + if: github.event_name == 'push' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository) strategy: fail-fast: false matrix: @@ -49,7 +52,7 @@ jobs: sqlite_url: "https://www.sqlite.org/2022/sqlite-autoconf-3380000.tar.gz" dart_sdk: stable steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v5 - uses: dart-lang/setup-dart@v1 with: sdk: ${{ matrix.dart_sdk }} From 0234d4fa469702f2709d6933ba199ffac9a19941 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Mon, 29 Sep 2025 16:07:40 +0200 Subject: [PATCH 88/90] Format --- packages/drift_sqlite_async/test/generated/database.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/drift_sqlite_async/test/generated/database.dart b/packages/drift_sqlite_async/test/generated/database.dart index b8a5109..4e03600 100644 --- a/packages/drift_sqlite_async/test/generated/database.dart +++ b/packages/drift_sqlite_async/test/generated/database.dart @@ -15,7 +15,7 @@ class TodoItems extends Table { @DriftDatabase(tables: [TodoItems]) class TodoDatabase extends _$TodoDatabase { TodoDatabase(SqliteConnection db) : super(SqliteAsyncDriftConnection(db)); - + TodoDatabase.fromSqliteAsyncConnection(SqliteAsyncDriftConnection super.conn); @override From b97be616b5ab5095480cf71848a74d812ee74fac Mon Sep 17 00:00:00 2001 From: David Martos Date: Tue, 7 Oct 2025 16:26:06 +0100 Subject: [PATCH 89/90] Get all Sqlite connections in the pool (#101) --- .../lib/src/common/sqlite_database.dart | 8 + .../src/impl/single_connection_database.dart | 8 + .../lib/src/impl/stub_sqlite_database.dart | 8 + .../src/native/database/connection_pool.dart | 70 ++++++++ .../database/native_sqlite_database.dart | 8 + .../sqlite_async/lib/src/web/database.dart | 8 + .../src/web/database/web_sqlite_database.dart | 8 + packages/sqlite_async/test/basic_test.dart | 44 +++++ .../sqlite_async/test/native/basic_test.dart | 152 ++++++++++++++++++ 9 files changed, 314 insertions(+) diff --git a/packages/sqlite_async/lib/src/common/sqlite_database.dart b/packages/sqlite_async/lib/src/common/sqlite_database.dart index 3cb12bb..3201135 100644 --- a/packages/sqlite_async/lib/src/common/sqlite_database.dart +++ b/packages/sqlite_async/lib/src/common/sqlite_database.dart @@ -39,6 +39,14 @@ mixin SqliteDatabaseMixin implements SqliteConnection, SqliteQueries { /// /// Use this to access the database in background isolates. IsolateConnectionFactory isolateConnectionFactory(); + + /// Locks all underlying connections making up this database, and gives [block] access to all of them at once. + /// This can be useful to run the same statement on all connections. For instance, + /// ATTACHing a database, that is expected to be available in all connections. + Future withAllConnections( + Future Function( + SqliteWriteContext writer, List readers) + block); } /// A SQLite database instance. diff --git a/packages/sqlite_async/lib/src/impl/single_connection_database.dart b/packages/sqlite_async/lib/src/impl/single_connection_database.dart index 4cd3144..7ca4357 100644 --- a/packages/sqlite_async/lib/src/impl/single_connection_database.dart +++ b/packages/sqlite_async/lib/src/impl/single_connection_database.dart @@ -57,4 +57,12 @@ final class SingleConnectionDatabase return connection.writeLock(callback, lockTimeout: lockTimeout, debugContext: debugContext); } + + @override + Future withAllConnections( + Future Function( + SqliteWriteContext writer, List readers) + block) { + return writeLock((_) => block(connection, [])); + } } diff --git a/packages/sqlite_async/lib/src/impl/stub_sqlite_database.dart b/packages/sqlite_async/lib/src/impl/stub_sqlite_database.dart index 29db641..ee254f3 100644 --- a/packages/sqlite_async/lib/src/impl/stub_sqlite_database.dart +++ b/packages/sqlite_async/lib/src/impl/stub_sqlite_database.dart @@ -64,4 +64,12 @@ class SqliteDatabaseImpl Future getAutoCommit() { throw UnimplementedError(); } + + @override + Future withAllConnections( + Future Function( + SqliteWriteContext writer, List readers) + block) { + throw UnimplementedError(); + } } diff --git a/packages/sqlite_async/lib/src/native/database/connection_pool.dart b/packages/sqlite_async/lib/src/native/database/connection_pool.dart index 9521b34..8dab27e 100644 --- a/packages/sqlite_async/lib/src/native/database/connection_pool.dart +++ b/packages/sqlite_async/lib/src/native/database/connection_pool.dart @@ -31,6 +31,8 @@ class SqliteConnectionPool with SqliteQueries implements SqliteConnection { final MutexImpl mutex; + int _runningWithAllConnectionsCount = 0; + @override bool closed = false; @@ -88,6 +90,14 @@ class SqliteConnectionPool with SqliteQueries implements SqliteConnection { return; } + if (_availableReadConnections.isEmpty && + _runningWithAllConnectionsCount > 0) { + // Wait until [withAllConnections] is done. Otherwise we could spawn a new + // reader while the user is configuring all the connections, + // e.g. a global open factory configuration shared across all connections. + return; + } + var nextItem = _queue.removeFirst(); while (nextItem.completer.isCompleted) { // This item already timed out - try the next one if available @@ -232,6 +242,66 @@ class SqliteConnectionPool with SqliteQueries implements SqliteConnection { await connection.refreshSchema(); } } + + Future withAllConnections( + Future Function( + SqliteWriteContext writer, List readers) + block) async { + try { + _runningWithAllConnectionsCount++; + + final blockCompleter = Completer(); + final (write, reads) = await _lockAllConns(blockCompleter); + + try { + final res = await block(write, reads); + blockCompleter.complete(res); + return res; + } catch (e, st) { + blockCompleter.completeError(e, st); + rethrow; + } + } finally { + _runningWithAllConnectionsCount--; + + // Continue processing any pending read requests that may have been queued while + // the block was running. + Timer.run(_nextRead); + } + } + + /// Locks all connections, returning the acquired contexts. + /// We pass a completer that would be called after the locks are taken. + Future<(SqliteWriteContext, List)> _lockAllConns( + Completer lockCompleter) async { + final List> readLockedCompleters = []; + final Completer writeLockedCompleter = Completer(); + + // Take the write lock + writeLock((ctx) { + writeLockedCompleter.complete(ctx); + return lockCompleter.future; + }); + + // Take all the read locks + for (final readConn in _allReadConnections) { + final completer = Completer(); + readLockedCompleters.add(completer); + + readConn.readLock((ctx) { + completer.complete(ctx); + return lockCompleter.future; + }); + } + + // Wait after all locks are taken + final [writer as SqliteWriteContext, ...readers] = await Future.wait([ + writeLockedCompleter.future, + ...readLockedCompleters.map((e) => e.future) + ]); + + return (writer, readers); + } } typedef ReadCallback = Future Function(SqliteReadContext tx); diff --git a/packages/sqlite_async/lib/src/native/database/native_sqlite_database.dart b/packages/sqlite_async/lib/src/native/database/native_sqlite_database.dart index 7bea111..22cacf3 100644 --- a/packages/sqlite_async/lib/src/native/database/native_sqlite_database.dart +++ b/packages/sqlite_async/lib/src/native/database/native_sqlite_database.dart @@ -171,4 +171,12 @@ class SqliteDatabaseImpl Future refreshSchema() { return _pool.refreshSchema(); } + + @override + Future withAllConnections( + Future Function( + SqliteWriteContext writer, List readers) + block) { + return _pool.withAllConnections(block); + } } diff --git a/packages/sqlite_async/lib/src/web/database.dart b/packages/sqlite_async/lib/src/web/database.dart index cfaf987..f2dc998 100644 --- a/packages/sqlite_async/lib/src/web/database.dart +++ b/packages/sqlite_async/lib/src/web/database.dart @@ -171,6 +171,14 @@ class WebDatabase await isInitialized; return _database.fileSystem.flush(); } + + @override + Future withAllConnections( + Future Function( + SqliteWriteContext writer, List readers) + block) { + return writeLock((_) => block(this, [])); + } } final class _UnscopedContext extends UnscopedContext { diff --git a/packages/sqlite_async/lib/src/web/database/web_sqlite_database.dart b/packages/sqlite_async/lib/src/web/database/web_sqlite_database.dart index c6d1b75..69f01ab 100644 --- a/packages/sqlite_async/lib/src/web/database/web_sqlite_database.dart +++ b/packages/sqlite_async/lib/src/web/database/web_sqlite_database.dart @@ -178,4 +178,12 @@ class SqliteDatabaseImpl Future exposeEndpoint() async { return await _connection.exposeEndpoint(); } + + @override + Future withAllConnections( + Future Function( + SqliteWriteContext writer, List readers) + block) { + return writeLock((_) => block(_connection, [])); + } } diff --git a/packages/sqlite_async/test/basic_test.dart b/packages/sqlite_async/test/basic_test.dart index 6a315da..e2914b3 100644 --- a/packages/sqlite_async/test/basic_test.dart +++ b/packages/sqlite_async/test/basic_test.dart @@ -7,6 +7,7 @@ import 'utils/test_utils_impl.dart'; final testUtils = TestUtils(); const _isDart2Wasm = bool.fromEnvironment('dart.tool.dart2wasm'); +const _isWeb = identical(0, 0.0) || _isDart2Wasm; void main() { group('Shared Basic Tests', () { @@ -301,6 +302,49 @@ void main() { 'Web locks are managed with a shared worker, which does not support timeouts', ) }); + + test('with all connections', () async { + final maxReaders = _isWeb ? 0 : 3; + + final db = SqliteDatabase.withFactory( + await testUtils.testFactory(path: path), + maxReaders: maxReaders, + ); + await db.initialize(); + await createTables(db); + + // Warm up to spawn the max readers + await Future.wait([for (var i = 0; i < 10; i++) db.get('SELECT $i')]); + + bool finishedWithAllConns = false; + + late Future readsCalledWhileWithAllConnsRunning; + + final parentZone = Zone.current; + await db.withAllConnections((writer, readers) async { + expect(readers.length, maxReaders); + + // Run some reads during the block that they should run after the block finishes and releases + // all locks + // Need a root zone here to avoid recursive lock errors. + readsCalledWhileWithAllConnsRunning = + Future(parentZone.bindCallback(() async { + await Future.wait( + [1, 2, 3, 4, 5, 6, 7, 8].map((i) async { + await db.readLock((c) async { + expect(finishedWithAllConns, isTrue); + await Future.delayed(const Duration(milliseconds: 100)); + }); + }), + ); + })); + + await Future.delayed(const Duration(milliseconds: 200)); + finishedWithAllConns = true; + }); + + await readsCalledWhileWithAllConnsRunning; + }); }); } diff --git a/packages/sqlite_async/test/native/basic_test.dart b/packages/sqlite_async/test/native/basic_test.dart index dec1fed..3f348e6 100644 --- a/packages/sqlite_async/test/native/basic_test.dart +++ b/packages/sqlite_async/test/native/basic_test.dart @@ -2,12 +2,16 @@ library; import 'dart:async'; +import 'dart:io'; import 'dart:math'; +import 'package:collection/collection.dart'; +import 'package:path/path.dart' show join; import 'package:sqlite3/common.dart' as sqlite; import 'package:sqlite_async/sqlite_async.dart'; import 'package:test/test.dart'; +import '../utils/abstract_test_utils.dart'; import '../utils/test_utils_impl.dart'; final testUtils = TestUtils(); @@ -100,6 +104,126 @@ void main() { print("${DateTime.now()} done"); }); + test('prevent opening new readers while in withAllConnections', () async { + final sharedStateDir = Directory.systemTemp.createTempSync(); + addTearDown(() => sharedStateDir.deleteSync(recursive: true)); + + final File sharedStateFile = + File(join(sharedStateDir.path, 'shared-state.txt')); + + sharedStateFile.writeAsStringSync('initial'); + + final db = SqliteDatabase.withFactory( + _TestSqliteOpenFactoryWithSharedStateFile( + path: path, sharedStateFilePath: sharedStateFile.path), + maxReaders: 3); + await db.initialize(); + await createTables(db); + + // The writer saw 'initial' in the file when opening the connection + expect( + await db + .writeLock((c) => c.get('SELECT file_contents_on_open() AS state')), + {'state': 'initial'}, + ); + + final withAllConnectionsCompleter = Completer(); + + final withAllConnsFut = db.withAllConnections((writer, readers) async { + expect(readers.length, 0); // No readers yet + + // Simulate some work until the file is updated + await Future.delayed(const Duration(milliseconds: 200)); + sharedStateFile.writeAsStringSync('updated'); + + await withAllConnectionsCompleter.future; + }); + + // Start a reader that gets the contents of the shared file + bool readFinished = false; + final someReadFut = + db.get('SELECT file_contents_on_open() AS state', []).then((r) { + readFinished = true; + return r; + }); + + // The withAllConnections should prevent the reader from opening + await Future.delayed(const Duration(milliseconds: 100)); + expect(readFinished, isFalse); + + // Free all the locks + withAllConnectionsCompleter.complete(); + await withAllConnsFut; + + final readerInfo = await someReadFut; + expect(readFinished, isTrue); + // The read should see the updated value in the file. This checks + // that a reader doesn't spawn while running withAllConnections + expect(readerInfo, {'state': 'updated'}); + }); + + test('with all connections', () async { + final maxReaders = 3; + + final db = SqliteDatabase.withFactory( + await testUtils.testFactory(path: path), + maxReaders: maxReaders, + ); + await db.initialize(); + await createTables(db); + + Future readWithRandomDelay( + SqliteReadContext ctx, int id) async { + return await ctx.get( + 'SELECT ? as i, test_sleep(?) as sleep, test_connection_name() as connection', + [id, 5 + Random().nextInt(10)]); + } + + // Warm up to spawn the max readers + await Future.wait( + [1, 2, 3, 4, 5, 6, 7, 8].map((i) => readWithRandomDelay(db, i)), + ); + + bool finishedWithAllConns = false; + + late Future readsCalledWhileWithAllConnsRunning; + + print("${DateTime.now()} start"); + await db.withAllConnections((writer, readers) async { + expect(readers.length, maxReaders); + + // Run some reads during the block that they should run after the block finishes and releases + // all locks + readsCalledWhileWithAllConnsRunning = Future.wait( + [1, 2, 3, 4, 5, 6, 7, 8].map((i) async { + final r = await db.readLock((c) async { + expect(finishedWithAllConns, isTrue); + return await readWithRandomDelay(c, i); + }); + print( + "${DateTime.now()} After withAllConnections, started while running $r"); + }), + ); + + await Future.wait([ + writer.execute( + "INSERT OR REPLACE INTO test_data(id, description) SELECT ? as i, test_sleep(?) || ' ' || test_connection_name() || ' 1 ' || datetime() as connection RETURNING *", + [ + 123, + 5 + Random().nextInt(20) + ]).then((value) => + print("${DateTime.now()} withAllConnections writer done $value")), + ...readers + .mapIndexed((i, r) => readWithRandomDelay(r, i).then((results) { + print( + "${DateTime.now()} withAllConnections readers done $results"); + })) + ]); + }).then((_) => finishedWithAllConns = true); + + await readsCalledWhileWithAllConnsRunning; + }); + test('read-only transactions', () async { final db = await testUtils.setupDatabase(path: path); await createTables(db); @@ -379,3 +503,31 @@ class _InvalidPragmaOnOpenFactory extends DefaultSqliteOpenFactory { ]; } } + +class _TestSqliteOpenFactoryWithSharedStateFile + extends TestDefaultSqliteOpenFactory { + final String sharedStateFilePath; + + _TestSqliteOpenFactoryWithSharedStateFile( + {required super.path, required this.sharedStateFilePath}); + + @override + sqlite.CommonDatabase open(SqliteOpenOptions options) { + final File sharedStateFile = File(sharedStateFilePath); + final String sharedState = sharedStateFile.readAsStringSync(); + + final db = super.open(options); + + // Function to return the contents of the shared state file at the time of opening + // so that we know at which point the factory was called. + db.createFunction( + functionName: 'file_contents_on_open', + argumentCount: const sqlite.AllowedArgumentCount(0), + function: (args) { + return sharedState; + }, + ); + + return db; + } +} From 21df6da19d3e69762047a638c1e6fef8b710e713 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Mon, 13 Oct 2025 17:25:46 +0200 Subject: [PATCH 90/90] chore(release): publish packages - sqlite_async@0.12.2 - drift_sqlite_async@0.2.5 --- CHANGELOG.md | 26 ++++++++++++++++++++++++ packages/drift_sqlite_async/CHANGELOG.md | 4 ++++ packages/drift_sqlite_async/pubspec.yaml | 4 ++-- packages/sqlite_async/CHANGELOG.md | 4 ++++ packages/sqlite_async/pubspec.yaml | 2 +- 5 files changed, 37 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 821f8db..5181152 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,32 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## 2025-10-13 + +### Changes + +--- + +Packages with breaking changes: + + - There are no breaking changes in this release. + +Packages with other changes: + + - [`sqlite_async` - `v0.12.2`](#sqlite_async---v0122) + - [`drift_sqlite_async` - `v0.2.5`](#drift_sqlite_async---v025) + +--- + +#### `sqlite_async` - `v0.12.2` + + - Add `withAllConnections` method to run statements on all connections in the pool. + +#### `drift_sqlite_async` - `v0.2.5` + + - Allow customizing update notifications from `sqlite_async`. + + ## 2025-08-08 ### Changes diff --git a/packages/drift_sqlite_async/CHANGELOG.md b/packages/drift_sqlite_async/CHANGELOG.md index 8dd5e5a..bd36dd1 100644 --- a/packages/drift_sqlite_async/CHANGELOG.md +++ b/packages/drift_sqlite_async/CHANGELOG.md @@ -1,3 +1,7 @@ +## 0.2.5 + + - Allow customizing update notifications from `sqlite_async`. + ## 0.2.4 - Allow transforming table updates from sqlite_async. diff --git a/packages/drift_sqlite_async/pubspec.yaml b/packages/drift_sqlite_async/pubspec.yaml index 06e2963..bd2b701 100644 --- a/packages/drift_sqlite_async/pubspec.yaml +++ b/packages/drift_sqlite_async/pubspec.yaml @@ -1,5 +1,5 @@ name: drift_sqlite_async -version: 0.2.4 +version: 0.2.5 homepage: https://github.com/powersync-ja/sqlite_async.dart repository: https://github.com/powersync-ja/sqlite_async.dart description: Use Drift with a sqlite_async database, allowing both to be used in the same application. @@ -15,7 +15,7 @@ environment: sdk: ">=3.0.0 <4.0.0" dependencies: drift: ">=2.28.0 <3.0.0" - sqlite_async: ^0.12.0 + sqlite_async: ^0.12.2 dev_dependencies: build_runner: ^2.4.8 diff --git a/packages/sqlite_async/CHANGELOG.md b/packages/sqlite_async/CHANGELOG.md index 342adca..1bbc939 100644 --- a/packages/sqlite_async/CHANGELOG.md +++ b/packages/sqlite_async/CHANGELOG.md @@ -1,3 +1,7 @@ +## 0.12.2 + + - Add `withAllConnections` method to run statements on all connections in the pool. + ## 0.12.1 - Fix distributing updates from shared worker. diff --git a/packages/sqlite_async/pubspec.yaml b/packages/sqlite_async/pubspec.yaml index 4b45d57..3d5130d 100644 --- a/packages/sqlite_async/pubspec.yaml +++ b/packages/sqlite_async/pubspec.yaml @@ -1,6 +1,6 @@ name: sqlite_async description: High-performance asynchronous interface for SQLite on Dart and Flutter. -version: 0.12.1 +version: 0.12.2 repository: https://github.com/powersync-ja/sqlite_async.dart environment: sdk: ">=3.5.0 <4.0.0"