diff --git a/docs/content-migrations.md b/docs/content-migrations.md index 3350ea169..9d59671c0 100644 --- a/docs/content-migrations.md +++ b/docs/content-migrations.md @@ -165,6 +165,6 @@ With `Auto`, startup skips the content stream and applies additive changes by co | Check constraint of a new JSON column | Reported, because SQLite adds one only by rebuilding the table | | Dropped table, column or index; changed column type, nullability or index | Reported | -Reported changes are logged as warnings and never applied. To get them, recreate the development database. Instances that start together on one database apply each change once: an instance with something to apply takes the `fluentcms:schema-sync` lock and compares again before it applies. It waits up to 15 minutes for the lock and then fails to start with an error that names it. Check the `owner` and `lease_until` of the lock's row in `cms_lock`: a `lease_until` that keeps moving means its holder is still working, and one that stands still ahead of the instance's clock means its holder stopped and the clocks of the instances are not in step. On SQLite, one that stands still behind the clock can also mean that its holder is working inside a long write transaction, which keeps it from renewing until the transaction ends. Once no instance is running, set the row's `owner` to null. A table under a plugin's prefix (`cms_{plugin}_`) belongs to that plugin's stream, so schema sync never reports it as a dropped table. That holds for a plugin the host no longer adds too, while the database has the plugin's history table `cms_{plugin}__migrations`. For a host without content migrations, schema sync records the schema history only when it applied every change and the history counts none as lost, such as a removed type or a field renamed whose values are kept in JSON; such a change keeps the newest row for `fluentcms schema apply`. A host with content migrations records after every sync that left nothing undone, as before. +Reported changes are logged as warnings and never applied. To get them, recreate the development database. Instances that start together on one database apply each change once: an instance with something to apply takes the `fluentcms:schema-sync` lock and compares again before it applies. It waits up to 15 minutes for the lock and then fails to start with an error that names it. Check the `owner` and `lease_until` of the lock's row in `cms_lock`: a `lease_until` that keeps moving means its holder is still working, and one that stands still ahead of the instance's clock means its holder stopped and the clocks of the instances are not in step. On SQLite, one that stands still behind the clock can also mean that its holder is working inside a long write transaction, which keeps it from renewing until the transaction ends. When the log has event 9166 for the lock, it can also mean that the database was busy: another connection kept SQLite's one writer to the end of the wait, and the row may be free. Once no instance is running, set the row's `owner` to null. A table under a plugin's prefix (`cms_{plugin}_`) belongs to that plugin's stream, so schema sync never reports it as a dropped table. That holds for a plugin the host no longer adds too, while the database has the plugin's history table `cms_{plugin}__migrations`. For a host without content migrations, schema sync records the schema history only when it applied every change and the history counts none as lost, such as a removed type or a field renamed whose values are kept in JSON; such a change keeps the newest row for `fluentcms schema apply`. A host with content migrations records after every sync that left nothing undone, as before. A database that schema sync has changed has no content history, so the first migration of the content stream will not apply to it. Switch to `Off` on a new database. diff --git a/docs/planning/backend-technical-spec.md b/docs/planning/backend-technical-spec.md index ba29f086a..5960a2db2 100644 --- a/docs/planning/backend-technical-spec.md +++ b/docs/planning/backend-technical-spec.md @@ -1165,11 +1165,11 @@ Rows 10.3, 10.5 to 10.8, 10.11 to 10.15. **Observability.** One `ActivitySource` (`FluentCMS`) with spans for content operations, hooks, jobs, webhooks and MCP calls; one `Meter` with request counts by surface, query duration, job latency, cache hit ratio, webhook outcomes; structured logs with event ids and `FCMS` codes. Exporters are the host's choice. Nothing is sent to the project. -**Health and observability, as built.** `FluentCmsHealthCheckTags` (`FluentCMS.Diagnostics`, in the SDK, so plugins tag their checks) has two tags, `live` and `ready`, and `/health/live` and `/health/ready` run the checks tagged with each through `Microsoft.Extensions.Diagnostics.HealthChecks`, with no authentication and no rate limit policy of the endpoints. They stay reachable without credentials, because the image's `HEALTHCHECK` and an orchestrator's probes call them, so they are kept cheap and bounded instead: the FluentCMS authentication step skips the `health` surface, so a key sent there is never looked up, and the check before authentication counts them per client address in a partition of their own, apart from the coarse limit, with `FluentCms:RateLimits:Health:PermitLimit` (1200 a minute, twenty a second from one address, far above what an orchestrator sends one instance). A path under `/health` that no endpoint answers counts there too. A refusal is the same 429 problem, and the probes carry no `RateLimit-*` headers. Where the network allows, probes should still be restricted to the orchestrator and the load balancer. This answers decision 16 of the review of 7 Nov 2026 ([#2847](https://github.com/fluentcms/FluentCMS/issues/2847), [#2855](https://github.com/fluentcms/FluentCMS/issues/2855)). They answer 200 for Healthy and Degraded and 503 for Unhealthy, with the status as the only body, never a description or an exception. Core registers `process` (live), `database` (ready, one row of a system table, so it also fails when the system migrations did not run) and `job-runner` (ready); `services.AddHealthChecks().AddFluentCmsReadyCheck("name")` adds a ready check with a five second timeout (it is in Core, because the SDK has no reference to the health check abstractions; a plugin that references only the SDK calls `AddCheck` with the tag). The job runner beats in memory at every turn of its maintenance loop (`JobRunnerOptions.MaintenanceInterval`, 15 seconds), and ready is Unhealthy when the last beat is older than `FluentCms:Health:JobRunnerHeartbeatTimeout` (default 2 minutes; `Timeout.InfiniteTimeSpan` leaves it out for an instance that runs no jobs) or when the runner has not started, and Degraded when the oldest pending job of a queue has waited longer than `FluentCms:Health:PendingJobAgeLimit` (4.12). `GET /api/management/v1/system/health` needs `system.read`, an instance-level permission: it is in the catalogue and checked with `InstanceAccess`, so a role of one site or a grant limited to sites does not reach it, nor does a key bound to sites or a Delivery or Preview key. It runs every check and answers 200 with `{ status, totalDurationMs, checks: [{ name, status, durationMs, tags, description, error }] }`, whatever the status; descriptions and exception text pass through `SecretScrubber`. The one `ActivitySource` and the one `Meter` are both named `FluentCMS` (`FluentCmsTelemetry.Name`), and nothing is exported: a host subscribes with `AddSource` and `AddMeter` from its own OpenTelemetry setup, and Core takes no dependency on it. Spans are named `content {operation}` (create, update, patch, delete, restore, purge, publish, unpublish, discard-draft, schedule, cancel-schedule, versions, version, restore-version, diff, get, query, query-trash, count, bulk), `hook {kind}` (one per run of the hooks of a kind, with `fluentcms.hooks` their number) and `job {type}` (with `fluentcms.queue`). The MCP plugin adds `mcp {tool}` for a tool call (5.4). They carry `fluentcms.operation`, `fluentcms.type` (the name the schema resolves, and not on jobs) and `fluentcms.site`, and never a value of content; a job's span has the Error status when the attempt throws. The instruments are `fluentcms.requests` (counter, by `fluentcms.surface` of delivery, management, connect (with `/.well-known`), account, health or docs (the OpenAPI document and Scalar), and `fluentcms.status_class`, counted with the status the client got, including a refusal an endpoint throws), `fluentcms.query.duration` (histogram, seconds, by the `fluentcms.type` the schema resolved, and not recorded for a query refused as invalid), `fluentcms.job.latency` (histogram, seconds, the length of an attempt by `fluentcms.operation` the job type and `fluentcms.outcome`), `fluentcms.job.oldest_pending_age` (observable gauge, seconds, by `fluentcms.queue`, 4.12), `fluentcms.event.handler.duration` (histogram, seconds, the length of a run of an in-process event handler by `fluentcms.operation` the event type, `fluentcms.handler` and `fluentcms.outcome`, 4.11), and `fluentcms.cache.requests` (counter by `fluentcms.cache` and `fluentcms.result`) with `fluentcms.cache.hit_ratio` (observable gauge by `fluentcms.cache`); the Delivery output cache reports its lookups as `delivery`. Every `[LoggerMessage]` in Core has an event id, in a range per area that follows the `FCMS` families: 90xx identity (the authorization server 9001 to 9003 and 9005, the hosted pages 9004, admin SSO 9040 to 9044, the member pool 9070 to 9076, member providers 9090 to 9094), 91xx host (health 9101 to 9104, the problem handler 9110, email 9120, the startup report 9130 to 9132, the distributed lock 9160 to 9165 of the range 9160 to 9169, the command runner 9170 to 9172 of the range 9170 to 9179), 92xx plugins (per-site enablement 9230 to 9234), 93xx storage (migrations 9301 to 9305, schema history 9311), 94xx content (9401, scheduled jobs 9411, trash purge 9421), 95xx jobs, 96xx events, 97xx activity, 98xx settings and 99xx seeding. The event name is the method name, and a message carries the `FCMS` code where one applies. +**Health and observability, as built.** `FluentCmsHealthCheckTags` (`FluentCMS.Diagnostics`, in the SDK, so plugins tag their checks) has two tags, `live` and `ready`, and `/health/live` and `/health/ready` run the checks tagged with each through `Microsoft.Extensions.Diagnostics.HealthChecks`, with no authentication and no rate limit policy of the endpoints. They stay reachable without credentials, because the image's `HEALTHCHECK` and an orchestrator's probes call them, so they are kept cheap and bounded instead: the FluentCMS authentication step skips the `health` surface, so a key sent there is never looked up, and the check before authentication counts them per client address in a partition of their own, apart from the coarse limit, with `FluentCms:RateLimits:Health:PermitLimit` (1200 a minute, twenty a second from one address, far above what an orchestrator sends one instance). A path under `/health` that no endpoint answers counts there too. A refusal is the same 429 problem, and the probes carry no `RateLimit-*` headers. Where the network allows, probes should still be restricted to the orchestrator and the load balancer. This answers decision 16 of the review of 7 Nov 2026 ([#2847](https://github.com/fluentcms/FluentCMS/issues/2847), [#2855](https://github.com/fluentcms/FluentCMS/issues/2855)). They answer 200 for Healthy and Degraded and 503 for Unhealthy, with the status as the only body, never a description or an exception. Core registers `process` (live), `database` (ready, one row of a system table, so it also fails when the system migrations did not run) and `job-runner` (ready); `services.AddHealthChecks().AddFluentCmsReadyCheck("name")` adds a ready check with a five second timeout (it is in Core, because the SDK has no reference to the health check abstractions; a plugin that references only the SDK calls `AddCheck` with the tag). The job runner beats in memory at every turn of its maintenance loop (`JobRunnerOptions.MaintenanceInterval`, 15 seconds), and ready is Unhealthy when the last beat is older than `FluentCms:Health:JobRunnerHeartbeatTimeout` (default 2 minutes; `Timeout.InfiniteTimeSpan` leaves it out for an instance that runs no jobs) or when the runner has not started, and Degraded when the oldest pending job of a queue has waited longer than `FluentCms:Health:PendingJobAgeLimit` (4.12). `GET /api/management/v1/system/health` needs `system.read`, an instance-level permission: it is in the catalogue and checked with `InstanceAccess`, so a role of one site or a grant limited to sites does not reach it, nor does a key bound to sites or a Delivery or Preview key. It runs every check and answers 200 with `{ status, totalDurationMs, checks: [{ name, status, durationMs, tags, description, error }] }`, whatever the status; descriptions and exception text pass through `SecretScrubber`. The one `ActivitySource` and the one `Meter` are both named `FluentCMS` (`FluentCmsTelemetry.Name`), and nothing is exported: a host subscribes with `AddSource` and `AddMeter` from its own OpenTelemetry setup, and Core takes no dependency on it. Spans are named `content {operation}` (create, update, patch, delete, restore, purge, publish, unpublish, discard-draft, schedule, cancel-schedule, versions, version, restore-version, diff, get, query, query-trash, count, bulk), `hook {kind}` (one per run of the hooks of a kind, with `fluentcms.hooks` their number) and `job {type}` (with `fluentcms.queue`). The MCP plugin adds `mcp {tool}` for a tool call (5.4). They carry `fluentcms.operation`, `fluentcms.type` (the name the schema resolves, and not on jobs) and `fluentcms.site`, and never a value of content; a job's span has the Error status when the attempt throws. The instruments are `fluentcms.requests` (counter, by `fluentcms.surface` of delivery, management, connect (with `/.well-known`), account, health or docs (the OpenAPI document and Scalar), and `fluentcms.status_class`, counted with the status the client got, including a refusal an endpoint throws), `fluentcms.query.duration` (histogram, seconds, by the `fluentcms.type` the schema resolved, and not recorded for a query refused as invalid), `fluentcms.job.latency` (histogram, seconds, the length of an attempt by `fluentcms.operation` the job type and `fluentcms.outcome`), `fluentcms.job.oldest_pending_age` (observable gauge, seconds, by `fluentcms.queue`, 4.12), `fluentcms.event.handler.duration` (histogram, seconds, the length of a run of an in-process event handler by `fluentcms.operation` the event type, `fluentcms.handler` and `fluentcms.outcome`, 4.11), and `fluentcms.cache.requests` (counter by `fluentcms.cache` and `fluentcms.result`) with `fluentcms.cache.hit_ratio` (observable gauge by `fluentcms.cache`); the Delivery output cache reports its lookups as `delivery`. Every `[LoggerMessage]` in Core has an event id, in a range per area that follows the `FCMS` families: 90xx identity (the authorization server 9001 to 9003 and 9005, the hosted pages 9004, admin SSO 9040 to 9044, the member pool 9070 to 9076, member providers 9090 to 9094), 91xx host (health 9101 to 9104, the problem handler 9110, email 9120, the startup report 9130 to 9132, the distributed lock 9160 to 9166 of the range 9160 to 9169, the command runner 9170 to 9172 of the range 9170 to 9179), 92xx plugins (per-site enablement 9230 to 9234), 93xx storage (migrations 9301 to 9305, schema history 9311), 94xx content (9401, scheduled jobs 9411, trash purge 9421), 95xx jobs, 96xx events, 97xx activity, 98xx settings and 99xx seeding. The event name is the method name, and a message carries the `FCMS` code where one applies. **Multi-instance.** Stateless API; Data Protection and signing keys persisted to the database by default; jobs lease through the database; `IDistributedLock` defaults to a `cms_lock` table and gets a Redis provider in `FluentCMS.Caching.Redis`; output cache in memory by default or Redis; media on shared storage. -**Multi-instance as built.** `IDistributedLock` (`FluentCMS.Providers`, in the SDK) has one method, `TryAcquireAsync(name, timeout)`, which returns a handle that releases the lock when disposed, or null when the timeout passed first (`TimeSpan.Zero` tries once, `Timeout.InfiniteTimeSpan` waits until the lock is free); the handle's `Lost` token is canceled when the lock is lost while held. Core's default, added with `TryAdd` and with no options, keeps one row per name in `cms_lock (name, owner, acquired_at, lease_until)`. A holder takes a 30 second lease and renews it every 10 seconds while it holds the lock, so a holder that stopped without releasing it blocks the others until its lease runs out, and the next holder takes it over with a warning (event id 9160). Taking the lock is an update conditional on the row being free or its lease over, so of two that try at once one wins, and a waiter only reads the row while the lock is held, polling from every 50 milliseconds up to once a second. A free lock keeps its row with no owner, so only the first use of a name inserts, and lock names are a fixed set, one per task rather than one per item. A renewal that finds another holder, or that fails until the lease has run out, cancels `Lost` (9161); a renewal or a release that fails is logged (9162, 9163), and a release changes nothing once another holder has the lock. SQLite has one writer, so while another connection keeps a write transaction open, such as the schema apply or schema sync that holds the lock, a renewal waits for it and gives up after the busy timeout and the retries of Microsoft.Data.Sqlite, about 35 seconds, and so does a take; in the rollback journal mode a read of the row can wait too. Such a command, which `IDataProvider.IsBusy` recognises (SQLite's busy and locked errors by their primary code, never on PostgreSQL or SQL Server), changed nothing: a waiter counts the lock as held and ends with the lock or its timeout, which it can overshoot by one such wait, and the holder reads the row, which in WAL mode does not wait, and keeps the lock while the row still names it, renewing again at once and logging once that it waits (9165). A take and a renewal each run in a transaction of their own and read the clock once it has begun, so on SQLite, where beginning it waits for the writer, a lease starts when the database let the write in, not when it was asked for; on PostgreSQL and SQL Server the clock is still read before a statement that can wait on the row, as before. A take that loses the insert of a name's first row rolls its transaction back before it reads the row, since a read on another connection of the in-memory database waits for it ([#3200](https://github.com/fluentcms/FluentCMS/issues/3200)). Before, a holder whose transaction lasted longer than about 45 seconds lost the lock, which canceled the rest of its work and rolled the transaction back, and a waiter failed with `SqliteException` ([#3176](https://github.com/fluentcms/FluentCMS/issues/3176)). One race remains: a waiter that found the lease over while such a transaction lasted waits for the writer beside the holder's renewal, and when the transaction ends it can come first. The renewal then finds the other holder and cancels `Lost`, but a holder with more than one unit of work under the lock may have work left: schema apply records the schema history after its change, and EF commits what came before a SQLite table rebuild. With a waiter present at the end of a long transaction, the waiter took the lock in 4 of 16 measured rounds, and the holder's `Lost` fired 64 to 119 milliseconds after the commit; when the holder began its next write at once, the waiter took it in none of 16. Known and left to [#3186](https://github.com/fluentcms/FluentCMS/issues/3186): a `schema apply` that loses the lock this way after its change ends with the generic `FCMS3302 ... TaskCanceledException`, not with a message that says to run it again, which records the history; and the waiter's event 9160 then says that a holder stopped without releasing the lock while that holder is alive. Leases are read against each instance's clock, so instances keep their clocks in step, as the job leases already need. The Data Protection ring is in `cms_data_protection_key` (4.7), so instances on one database read each other's cookies and protected payloads, and the keys survive a restart. Data Protection makes a key when its ring has none it can use, and instances that started together on an empty ring each made one; one that had read only its own kept it for up to a day, refusing the others' cookies once the two minutes in which Data Protection reloads the ring for an unknown key were over. The key manager now makes a key under the lock `fluentcms:data-protection-key`, after reading the ring again, and returns a key that another instance made in the last five minutes (the clock skew Data Protection allows) instead of making a second; when the lock is not free within 10 seconds it makes the key without it (9164). The wait blocks the thread Data Protection asks on, since `IKeyManager` is synchronous, so it covers a live holder, which makes its key within a second, and not one that stopped without releasing the lock, whose 30 second lease it used to wait out. A synchronous `TryAcquire` on `IDistributedLock` is declined: the default lock is a table read through EF Core's asynchronous calls, so it would only move the blocking, and every provider of the slot would have to implement it (item 6 of [#2887](https://github.com/fluentcms/FluentCMS/issues/2887)). The changes that can take Super Admin from its last holder run under `fluentcms:super-admins` (4.7), and fail rather than go on without it. The Delivery output cache is the `IOutputCacheStore` slot, in memory by default, which evicts only the copies of its own instance; a host that registers a shared store, such as `Microsoft.AspNetCore.OutputCaching.StackExchangeRedis`, evicts for every instance, and the key's schema hash keeps instances on different schemas apart (4.10). The member mail limit (4.7) is kept in the `IDistributedCache` slot, the framework's `MemoryDistributedCache` by default, one per instance, so N instances behind a load balancer send an address up to N mails a minute of each slot; a host with several instances registers a shared cache, such as `RedisCache`, and the startup report lists `member-mail-limit-per-instance` in Production while an email sender is set up and the cache is the in-memory one ([#3073](https://github.com/fluentcms/FluentCMS/issues/3073)). Each instance keeps its own copy of the sites and their locales, of plugin enablement and of settings, and sees a change another instance made within 30 seconds (4.6, 4.13); role changes are seen on the next request everywhere, since every request reads the role stamps (4.8). The Redis provider in `FluentCMS.Caching.Redis` is still to come. +**Multi-instance as built.** `IDistributedLock` (`FluentCMS.Providers`, in the SDK) has one method, `TryAcquireAsync(name, timeout)`, which returns a handle that releases the lock when disposed, or null when the timeout passed first (`TimeSpan.Zero` tries once, `Timeout.InfiniteTimeSpan` waits until the lock is free); the handle's `Lost` token is canceled when the lock is lost while held. Core's default, added with `TryAdd` and with no options, keeps one row per name in `cms_lock (name, owner, acquired_at, lease_until)`. A holder takes a 30 second lease and renews it every 10 seconds while it holds the lock, so a holder that stopped without releasing it blocks the others until its lease runs out, and the next holder takes it over with a warning (event id 9160). Taking the lock is an update conditional on the row being free or its lease over, so of two that try at once one wins, and a waiter only reads the row while the lock is held, polling from every 50 milliseconds up to once a second. A free lock keeps its row with no owner, so only the first use of a name inserts, and lock names are a fixed set, one per task rather than one per item. A renewal that finds another holder, or that fails until the lease has run out, cancels `Lost` (9161); a renewal or a release that fails is logged (9162, 9163), and a release changes nothing once another holder has the lock. SQLite has one writer, so while another connection keeps a write transaction open, such as the schema apply or schema sync that holds the lock, a renewal waits for it and gives up after the busy timeout and the retries of Microsoft.Data.Sqlite, about 35 seconds, and so does a take; in the rollback journal mode a read of the row can wait too. Such a command, which `IDataProvider.IsBusy` recognises (SQLite's busy and locked errors by their primary code, never on PostgreSQL or SQL Server), changed nothing: a waiter counts the lock as held and ends with the lock or its timeout, which it can overshoot by one such wait, and the holder reads the row, which in WAL mode does not wait, and keeps the lock while the row still names it, renewing again at once and logging once that it waits (9165). A take and a renewal each run in a transaction of their own and read the clock once it has begun, so on SQLite, where beginning it waits for the writer, a lease starts when the database let the write in, not when it was asked for; on PostgreSQL and SQL Server the clock is still read before a statement that can wait on the row, as before. A take that loses the insert of a name's first row rolls its transaction back before it reads the row, since a read on another connection of the in-memory database waits for it ([#3200](https://github.com/fluentcms/FluentCMS/issues/3200)). Before, a holder whose transaction lasted longer than about 45 seconds lost the lock, which canceled the rest of its work and rolled the transaction back, and a waiter failed with `SqliteException` ([#3176](https://github.com/fluentcms/FluentCMS/issues/3176)). A take reads the row again inside its transaction, and its update is conditional on that row, on the owner it read with a lease over or on no owner, so event 9160 names a holder only when the take took over that holder's lease; a holder that released the lock while the take waited for the writer leaves a free lock, which is taken with no warning. On a database with one writer (`IDataProvider.HasOneWriter`, true on SQLite only), a lease also runs out while a write transaction keeps a live holder's renewals waiting, and when the transaction ends a waiter that found the lease over can get the writer before the renewal: with a waiter present, it took the lock in 4 of 16 measured rounds, and the holder's `Lost` fired 64 to 119 milliseconds after the commit ([#3186](https://github.com/fluentcms/FluentCMS/issues/3186)). So there a take-over is confirmed, which narrows the race and does not close it. A take never takes over a lease the first time it sees it over: it answers that the lock is held, and its next try comes a second later and takes the lease only if it finds the same owner and `lease_until` in a transaction whose BEGIN, timed after the connection is open, took at most 50 milliseconds; a try whose BEGIN waited longer, or that gave up as busy, starts the confirmation again. A long transaction alone no longer lets a waiter in, since a take that waits for it starts again: with no other writer, the waiter took the live holder's lock in none of 44 and 20 measured rounds. Other writers still can. A renewal that has waited for the writer longer than about 228 milliseconds polls only every 100 milliseconds, while every fresh writer, the confirming take included, polls after 1, 2, 5, 10 and 15 milliseconds, so the window gives the waiter several tries against the renewal's one or none; a backlog of requests that queued behind the long transaction is that load. Measured on two hosts on one file with a holder lease of 1.5 seconds, the lease over inside a long write transaction and other writers running 5 millisecond transactions with 10 millisecond pauses, the waiter took the live holder's lock in 0 of 24 rounds with 2 or 4 writers, 3 of 55 with 8 and 2 of 23 with 16, and in a second probe of the same shape in 0 of 20 with 8 and 1 of 20 with 16; a second review counted 0 of 50 with no other writer, 3 of 50 with 8 and 7 of 50 with 16, where `main` before this change gave 22, 17 and 16 of 30. The 64 to 119 milliseconds after which `Lost` fired hold with no other writer; under write load the renewal that would notice the loss is starved by the same writers, and the second review saw `Lost` fire 63 milliseconds to 1.73 seconds after the take-over in 11 of 14 take-overs and not yet after 1.6 to 2.1 seconds in the other 3. Requiring three such tries spread over the second did not help (4 of 60 rounds with 16 writers, 3 of 40 with 32), nor would a longer confirmation under sustained load. The holder's own transactions back to back, such as a table rebuild, where a waiter could see the lease over in one gap and confirm it in another, let it in 0 of 30 rounds in each of two runs of the second review, against 12 and 19 of 30 on `main` before this change. The deterministic fix, the holder renewing its row inside its own transaction before it commits, changes the lock's callers and is left to [#3186](https://github.com/fluentcms/FluentCMS/issues/3186). A holder that stopped is taken over within about 2 seconds of its lease running out on a database whose writer is free (up to a second of polling to see it and the second of the confirmation), a second more for each try whose BEGIN waited; while other connections keep the writer it is not taken over, since a holder at work could not be told from it. A take-over being confirmed goes on for up to 5 seconds past the caller's timeout, so a caller with `TimeSpan.Zero`, such as an export or an import, still takes over a holder that stopped, a second late. Of two waiters that confirm at once, the conditional update lets one take the lock, and the other then finds it held. A free lock, a released one and the first use of a name are taken at once, and on PostgreSQL and SQL Server, where a renewal waits only for the row, a take-over is not confirmed, as before. A wait that reaches its timeout while its last try gave up as busy logs 9166, since the row may have been free behind another connection's transaction, and the error of the `fluentcms:schema-sync` timeout says that this can also be the cause. A `schema apply` that loses its lock, whatever canceled `Lost` (another holder took it, its lease ran out or the host stopped), says that the lock was lost, with the type of a provider's own exception when the canceled command ended with one, and ends with `FCMS3339` when that happened while it planned, which applied nothing, or while its operations ran, which rolls them back, and with `FCMS3340` when it happened after they committed and before the schema history was written; each says to run it again. Leases are read against each instance's clock, so instances keep their clocks in step, as the job leases already need. The Data Protection ring is in `cms_data_protection_key` (4.7), so instances on one database read each other's cookies and protected payloads, and the keys survive a restart. Data Protection makes a key when its ring has none it can use, and instances that started together on an empty ring each made one; one that had read only its own kept it for up to a day, refusing the others' cookies once the two minutes in which Data Protection reloads the ring for an unknown key were over. The key manager now makes a key under the lock `fluentcms:data-protection-key`, after reading the ring again, and returns a key that another instance made in the last five minutes (the clock skew Data Protection allows) instead of making a second; when the lock is not free within 10 seconds it makes the key without it (9164). The wait blocks the thread Data Protection asks on, since `IKeyManager` is synchronous, so it covers a live holder, which makes its key within a second, and not one that stopped without releasing the lock, whose 30 second lease it used to wait out. A synchronous `TryAcquire` on `IDistributedLock` is declined: the default lock is a table read through EF Core's asynchronous calls, so it would only move the blocking, and every provider of the slot would have to implement it (item 6 of [#2887](https://github.com/fluentcms/FluentCMS/issues/2887)). The changes that can take Super Admin from its last holder run under `fluentcms:super-admins` (4.7), and fail rather than go on without it. The Delivery output cache is the `IOutputCacheStore` slot, in memory by default, which evicts only the copies of its own instance; a host that registers a shared store, such as `Microsoft.AspNetCore.OutputCaching.StackExchangeRedis`, evicts for every instance, and the key's schema hash keeps instances on different schemas apart (4.10). The member mail limit (4.7) is kept in the `IDistributedCache` slot, the framework's `MemoryDistributedCache` by default, one per instance, so N instances behind a load balancer send an address up to N mails a minute of each slot; a host with several instances registers a shared cache, such as `RedisCache`, and the startup report lists `member-mail-limit-per-instance` in Production while an email sender is set up and the cache is the in-memory one ([#3073](https://github.com/fluentcms/FluentCMS/issues/3073)). Each instance keeps its own copy of the sites and their locales, of plugin enablement and of settings, and sees a change another instance made within 30 seconds (4.6, 4.13); role changes are seen on the next request everywhere, since every request reads the role stamps (4.8). The Redis provider in `FluentCMS.Caching.Redis` is still to come. **Reverse proxy.** Forwarded headers, `PathBase` option for sub-path hosting, HTTPS redirection and HSTS options; documented for Nginx, Caddy, YARP, Azure Front Door and Cloudflare. @@ -1689,7 +1689,7 @@ Inside the host, `FluentCMS.Core.Commands` has the one public type `FluentCmsCom Codes `FCMS3350` to `FCMS3359` are the host's and the tool's (an unknown role, site, recurring job or seeder, and a value that is not valid, are bad usage with exit 2), and `FCMS3361` to `FCMS3363` the tool's, the last a warning on standard error when a key goes over plain http to a machine other than this one; `FCMS3360` and `FCMS3364` to `FCMS3369` are free. A service's refusal inside the host prints each error with its code, as the API does, and exits 1. No migration, package or SDK type was added. -**Schema apply as built (B-98, [#3021](https://github.com/fluentcms/FluentCMS/issues/3021)).** `schema apply` is a command of the host and of the tool, which hands on `--script`, `--output` (made absolute), `--dry-run` and `--allow-destructive`, and refuses `--output` without `--script` and `--script` with `--dry-run`, as the host does (`FCMS3300`). It prints the changes from the newest row of the schema history to the host's schema in the words of `schema diff`, when the history has a row, then one line per operation, a destructive one marked as `schema diff` marks it, then under `Left alone:` each table of a plugin that the host does not register and each whose name has a control character (4.3), and ends with what it applied and the schema hash that the history now records. The operation and `Left alone:` lines write a control or bidi character of a name read from the database as its escape (`\u000A`, `PrintableText`), so in the script, where they are comments, a line break cannot end one. What it does is in 4.3. A host with content migrations in its set or in its history is `FCMS3335`; a pending system or plugin migration `FCMS3336`, which asks to run `fluentcms migrate` first; a change that loses data `FCMS3337`, refused unless `--allow-destructive` is given and then repeated as a warning; a required column with no empty value for the rows a table has `FCMS3338`; an operation the database refuses `FCMS3339`, after the transaction is rolled back; and a history row that cannot be written after the operations were applied `FCMS3340`. `--dry-run` exits with the code the run would have, so 1 when it would be refused. Codes `FCMS3335` to `FCMS3340`; `FCMS3341` to `FCMS3349` are free. No log event id was added: the operations are the command's output, and the history row logs 9311 as before. +**Schema apply as built (B-98, [#3021](https://github.com/fluentcms/FluentCMS/issues/3021)).** `schema apply` is a command of the host and of the tool, which hands on `--script`, `--output` (made absolute), `--dry-run` and `--allow-destructive`, and refuses `--output` without `--script` and `--script` with `--dry-run`, as the host does (`FCMS3300`). It prints the changes from the newest row of the schema history to the host's schema in the words of `schema diff`, when the history has a row, then one line per operation, a destructive one marked as `schema diff` marks it, then under `Left alone:` each table of a plugin that the host does not register and each whose name has a control character (4.3), and ends with what it applied and the schema hash that the history now records. The operation and `Left alone:` lines write a control or bidi character of a name read from the database as its escape (`\u000A`, `PrintableText`), so in the script, where they are comments, a line break cannot end one. What it does is in 4.3. A host with content migrations in its set or in its history is `FCMS3335`; a pending system or plugin migration `FCMS3336`, which asks to run `fluentcms migrate` first; a change that loses data `FCMS3337`, refused unless `--allow-destructive` is given and then repeated as a warning; a required column with no empty value for the rows a table has `FCMS3338`; an operation the database refuses `FCMS3339`, after the transaction is rolled back, and so does a lock lost while the operations are planned or run, which also says to run the command again; and a history row that cannot be written after the operations were applied, or whose lock was lost first, `FCMS3340`, which says to run it again to record the history. `--dry-run` exits with the code the run would have, so 1 when it would be refused. Codes `FCMS3335` to `FCMS3340`; `FCMS3341` to `FCMS3349` are free. No log event id was added: the operations are the command's output, and the history row logs 9311 as before. **Transfer and backup commands as built (B-101, [#3024](https://github.com/fluentcms/FluentCMS/issues/3024)).** `export` and `import` take `InstanceOptions` and `ProjectOptions`: with `--url` the tool calls the transfer endpoints itself (`TransferClient`, on the client the tool makes for an instance), and without it hands the command to the host with its paths made absolute (4.14). `backup` and `restore` are project commands only (4.15). The test that every command of the host is in the tool with the same options leaves `--url` and `--token` out of the comparison, as they are the tool's own. Tests give `CommandLine.RunAsync` a client of a test server for the instance. With `--url`, `export` writes only an answer that is `application/zip`, starts with a zip's signature and opens as one, and refuses anything else and a redirect with `FCMS3253`, leaving `--output` as it was (4.14, [#3067](https://github.com/fluentcms/FluentCMS/issues/3067)). The guide is `docs/backups.md`. diff --git a/src/FluentCMS.Core/Caching/DatabaseDistributedLock.cs b/src/FluentCMS.Core/Caching/DatabaseDistributedLock.cs index 61b260b15..bae97b32a 100644 --- a/src/FluentCMS.Core/Caching/DatabaseDistributedLock.cs +++ b/src/FluentCMS.Core/Caching/DatabaseDistributedLock.cs @@ -15,6 +15,18 @@ internal sealed class DatabaseDistributedLock(IServiceScopeFactory scopes, TimeP private static readonly TimeSpan FirstPoll = TimeSpan.FromMilliseconds(50); private static readonly TimeSpan LongestPoll = TimeSpan.FromSeconds(1); + // On a database with one writer: how long after a take saw a lease over its next try comes, which gives the holder's + // renewal, trying for the writer at least every 150 milliseconds while it waits, the time to get it when no other + // connection writes (a median of 80 milliseconds after the transaction, at most 660 in 50 measured rounds); and how long + // the beginning of a write transaction takes at most when it did not wait for another connection's. These are SQLite's + // and its driver's timings, kept here beside the lock that relies on them. + private static readonly TimeSpan Confirmation = TimeSpan.FromSeconds(1); + private static readonly TimeSpan LongBegin = TimeSpan.FromMilliseconds(50); + + // How long past its timeout a wait goes on confirming a take-over, so that a caller that tries once still takes the lock + // of a holder that stopped, even when a try of the confirmation waited and started it again. + private static readonly TimeSpan LateConfirmation = 5 * Confirmation; + // Who holds a lock, for whoever reads cms_lock; with a GUID per acquisition it stays within the owner column. private readonly string _instance = $"{Environment.MachineName[..Math.Min(Environment.MachineName.Length, 64)]}/{Environment.ProcessId}"; @@ -32,55 +44,97 @@ internal sealed class DatabaseDistributedLock(IServiceScopeFactory scopes, TimeP var holder = $"{_instance}/{Guid.NewGuid():N}"; var started = time.GetTimestamp(); var poll = FirstPoll; + Sighting? sighting = null; while (true) { - if (await TryTakeAsync(name, holder, cancellationToken) is { } leaseUntil) + var attempt = await TryTakeAsync(name, holder, sighting, cancellationToken); + if (attempt.Until is { } leaseUntil) { return new Handle(this, name, holder, leaseUntil); } - var remaining = timeout == Timeout.InfiniteTimeSpan ? poll : timeout - time.GetElapsedTime(started); - if (remaining <= TimeSpan.Zero) + sighting = attempt.Sighting; + var next = sighting is null ? poll : Confirmation; + var remaining = timeout == Timeout.InfiniteTimeSpan ? TimeSpan.MaxValue : timeout - time.GetElapsedTime(started); + if (remaining <= TimeSpan.Zero && (sighting is null || -remaining >= LateConfirmation)) { + if (attempt.Busy) + { + LockLog.TimedOutBusy(logger, name, timeout); + } + return null; } - await Task.Delay(remaining < poll ? remaining : poll, time, cancellationToken); - poll = poll * 2 < LongestPoll ? poll * 2 : LongestPoll; + // A confirmation keeps its length when the timeout comes first. + await Task.Delay(sighting is null && remaining < next ? remaining : next, time, cancellationToken); + if (sighting is null) + { + poll = poll * 2 < LongestPoll ? poll * 2 : LongestPoll; + } } } - // The end of the new lease, or null while another holder has the lock. On a database with one writer, reading the row or - // beginning the write can give up waiting for another connection's transaction: nothing was taken, and the lock counts - // as held, since that transaction may be the holder's own work, which keeps it from renewing. - private async Task TryTakeAsync(string name, string holder, CancellationToken cancellationToken) + // On a database with one writer, reading the row or beginning the write can give up waiting for another connection's + // transaction: nothing was taken, and the lock counts as held, since that transaction may be the holder's own work, which + // keeps it from renewing. + private async Task TryTakeAsync(string name, string holder, Sighting? sighting, CancellationToken cancellationToken) { await using var scope = scopes.CreateAsyncScope(); var context = scope.ServiceProvider.GetRequiredService(); + var provider = scope.ServiceProvider.GetRequiredService(); try { - return await TakeAsync(context, name, holder, cancellationToken); + return await TakeAsync(context, provider, name, holder, sighting, cancellationToken); } - catch (Exception exception) when (scope.ServiceProvider.GetRequiredService().IsBusy(exception)) + catch (Exception exception) when (provider.IsBusy(exception)) { - return null; + return new(null, null, Busy: true); } } - private async Task TakeAsync(SystemDbContext context, string name, string holder, CancellationToken cancellationToken) + private async Task TakeAsync(SystemDbContext context, IDataProvider provider, string name, string holder, Sighting? sighting, CancellationToken cancellationToken) { var current = await context.Set().AsNoTracking().SingleOrDefaultAsync(row => row.Name == name, cancellationToken); if (current is { Owner: not null } && current.LeaseUntil > time.GetUtcNow()) { - return null; + return default; } // The writes are not canceled, so a take that commits always comes back as a handle that renews and releases it. On // SQLite the transaction begins once the database's one writer is free, so the lease starts from the clock then. + // Opened first, so that only the beginning of the transaction is timed, and not the connection and its pragmas. + await context.Database.OpenConnectionAsync(cancellationToken); + var beginning = time.GetTimestamp(); await using var transaction = await context.Database.BeginTransactionAsync(CancellationToken.None); + var waited = time.GetElapsedTime(beginning) > LongBegin; var now = time.GetUtcNow(); var until = StorageConventions.ToMicroseconds(now + Lease); + // Read again inside the transaction, so that what the take writes and logs is the row as it is now: the holder may + // have renewed or released the lock while the transaction waited to begin. + current = await context.Set().AsNoTracking().SingleOrDefaultAsync(row => row.Name == name, cancellationToken); + if (current is { Owner: not null }) + { + if (current.LeaseUntil > now) + { + return default; + } + + // On a database with one writer, a lease runs out while a write transaction keeps the holder's renewals waiting, + // and when it ends this take can get the writer before the renewal does. So a take-over gives the renewal time to + // get the writer: it takes only a lease that its try before, a confirmation ago, saw over and as it is, in a + // transaction that began at once, and a transaction that waited starts the confirmation again. + if (provider.HasOneWriter) + { + var same = !waited && sighting is not null && sighting.Owner == current.Owner && sighting.LeaseUntil == current.LeaseUntil; + if (!same) + { + return new(null, new Sighting(current.Owner, current.LeaseUntil), Busy: false); + } + } + } + if (current is null) { context.Add(new LockRow { Name = name, Owner = holder, AcquiredAt = now, LeaseUntil = until }); @@ -88,7 +142,7 @@ internal sealed class DatabaseDistributedLock(IServiceScopeFactory scopes, TimeP { await context.SaveChangesAsync(CancellationToken.None); await transaction.CommitAsync(CancellationToken.None); - return until; + return new(until, null, Busy: false); } catch (DbUpdateException) { @@ -97,15 +151,19 @@ internal sealed class DatabaseDistributedLock(IServiceScopeFactory scopes, TimeP await transaction.RollbackAsync(CancellationToken.None); if (await ExistsAsync(name)) { - return null; + return default; } throw; } } - var taken = await context.Set() - .Where(row => row.Name == name && (row.Owner == null || row.LeaseUntil == null || row.LeaseUntil <= now)) + // Conditional on the row as read, so that a take-over that writes took over the lease of the holder it logs. + var rows = context.Set().Where(row => row.Name == name); + rows = current.Owner is { } owner + ? rows.Where(row => row.Owner == owner && (row.LeaseUntil == null || row.LeaseUntil <= now)) + : rows.Where(row => row.Owner == null); + var taken = await rows .ExecuteUpdateAsync( row => row .SetProperty(column => column.Owner, holder) @@ -116,7 +174,7 @@ internal sealed class DatabaseDistributedLock(IServiceScopeFactory scopes, TimeP if (taken == 0) { - return null; + return default; } if (current.Owner is not null) @@ -124,9 +182,16 @@ internal sealed class DatabaseDistributedLock(IServiceScopeFactory scopes, TimeP LockLog.LeaseTakenOver(logger, name, current.Owner); } - return until; + return new(until, null, Busy: false); } + // What a take found: the end of the lease it took; or, while it did not, a lease over that it saw and waits to confirm, and + // whether the database kept it waiting for another connection's transaction until it gave up. + private readonly record struct Attempt(DateTimeOffset? Until, Sighting? Sighting, bool Busy); + + // A lease over, as a take on a database with one writer saw it; the next try comes a confirmation later. + private sealed record Sighting(string Owner, DateTimeOffset? LeaseUntil); + private async Task ExistsAsync(string name) { await using var scope = scopes.CreateAsyncScope(); diff --git a/src/FluentCMS.Core/Caching/LockLog.cs b/src/FluentCMS.Core/Caching/LockLog.cs index bd6f00975..187a2c5c3 100644 --- a/src/FluentCMS.Core/Caching/LockLog.cs +++ b/src/FluentCMS.Core/Caching/LockLog.cs @@ -11,8 +11,9 @@ internal static partial class LockLog public const int ReleaseFailedId = 9163; public const int WentOnWithoutLockId = 9164; public const int RenewWaitingId = 9165; + public const int TimedOutBusyId = 9166; - [LoggerMessage(EventId = LeaseTakenOverId, EventName = "LockLeaseTakenOver", Level = LogLevel.Warning, Message = "Took the lock {Name} after the lease of {Holder} ran out; that holder stopped without releasing it.")] + [LoggerMessage(EventId = LeaseTakenOverId, EventName = "LockLeaseTakenOver", Level = LogLevel.Warning, Message = "Took the lock {Name} after the lease of {Holder} ran out; that holder stopped without releasing it or, on a database with one writer, could not renew it behind other writes.")] public static partial void LeaseTakenOver(ILogger logger, string name, string holder); [LoggerMessage(EventId = LostId, EventName = "LockLost", Level = LogLevel.Warning, Message = "Lost the lock {Name} while holding it, because {Reason}.")] @@ -29,4 +30,7 @@ internal static partial class LockLog [LoggerMessage(EventId = RenewWaitingId, EventName = "LockRenewWaiting", Level = LogLevel.Information, Message = "The lease of the lock {Name} is renewed once another connection's write transaction ends; the database has one writer, so no other holder can take the lock meanwhile.")] public static partial void RenewWaiting(ILogger logger, string name); + + [LoggerMessage(EventId = TimedOutBusyId, EventName = "LockTimedOutBusy", Level = LogLevel.Warning, Message = "The lock {Name} was not taken within {Timeout}, and the last try gave up waiting for another connection's write transaction; the database has one writer, so the lock may be free while that transaction lasts.")] + public static partial void TimedOutBusy(ILogger logger, string name, TimeSpan timeout); } diff --git a/src/FluentCMS.Core/Commands/SchemaApplyCommand.cs b/src/FluentCMS.Core/Commands/SchemaApplyCommand.cs index 53803d60e..7944ae5e9 100644 --- a/src/FluentCMS.Core/Commands/SchemaApplyCommand.cs +++ b/src/FluentCMS.Core/Commands/SchemaApplyCommand.cs @@ -3,6 +3,7 @@ using FluentCMS.Core.Events; using FluentCMS.Core.Schema; using FluentCMS.Core.Storage; +using FluentCMS.Providers; using FluentCMS.Schema; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.DependencyInjection; @@ -55,8 +56,20 @@ public override async Task RunAsync(CommandContext context, CancellationTok await using var held = applying ? await runner.LockContentSchemaAsync(cancellationToken) : null; using var guarded = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, held?.Lost ?? CancellationToken.None); - var applied = await runner.GetAppliedSnapshotAsync(guarded.Token); - var plan = await runner.PlanContentSchemaAsync(applied, guarded.Token); + string? applied; + SchemaApplyPlan plan; + try + { + applied = await runner.GetAppliedSnapshotAsync(guarded.Token); + plan = await runner.PlanContentSchemaAsync(applied, guarded.Token); + } + catch (Exception exception) when (LockLost(held, cancellationToken)) + { + throw new CommandException( + CommandErrorCodes.SchemaApplyFailed, + $"{Lost(exception)} while the operations were planned, so nothing was applied; run 'fluentcms schema apply' again"); + } + var steps = plan.Steps; var (report, unreadable) = SchemaHistory.Compare(applied, schema); if (!script) @@ -101,6 +114,13 @@ public override async Task RunAsync(CommandContext context, CancellationTok { await runner.ApplyContentSchemaAsync(steps, guarded.Token); } + // First, since a provider can end a command that the lost lock canceled with an exception of its own. + catch (Exception exception) when (LockLost(held, cancellationToken)) + { + throw new CommandException( + CommandErrorCodes.SchemaApplyFailed, + $"{Lost(exception)} while the operations were applied; their transaction was rolled back, except what SQLite committed before a table rebuild, which the next run leaves out, and the schema history keeps its last row; run 'fluentcms schema apply' again"); + } catch (DbException exception) { throw new CommandException( @@ -114,6 +134,12 @@ public override async Task RunAsync(CommandContext context, CancellationTok { await runner.RecordSchemaAsync(guarded.Token); } + catch (Exception exception) when (LockLost(held, cancellationToken)) + { + throw new CommandException( + CommandErrorCodes.SchemaNotRecorded, + $"the content tables match schema {hash}, but {Lost(exception)} before the schema history was written; run 'fluentcms schema apply' again with the same options to record it"); + } catch (Exception exception) when (exception is DbException or DbUpdateException or InvalidOperationException) { throw new CommandException( @@ -127,6 +153,15 @@ await context.Output.WriteLineAsync(steps.Count == 0 return CommandExitCodes.Success; } + // The lock's Lost canceled the work, and not the caller. + private static bool LockLost(IDistributedLockHandle? held, CancellationToken cancellationToken) => + held is { Lost.IsCancellationRequested: true } && !cancellationToken.IsCancellationRequested; + + // Lost is canceled for every reason the lock goes: another holder took it, its lease ran out, or the host stopped. A + // provider's own exception after it is named. + private static string Lost(Exception exception) => + $"the lock {MigrationRunner.SchemaSyncLock} was lost{(exception is OperationCanceledException ? "" : $" ({exception.GetType().Name})")}"; + private static async Task RefuseAsync(MigrationRunner runner, CancellationToken cancellationToken) { var migrations = await runner.GetContentMigrationsAsync(cancellationToken); diff --git a/src/FluentCMS.Core/Storage/IDataProvider.cs b/src/FluentCMS.Core/Storage/IDataProvider.cs index d079ca0d8..92123f4ec 100644 --- a/src/FluentCMS.Core/Storage/IDataProvider.cs +++ b/src/FluentCMS.Core/Storage/IDataProvider.cs @@ -36,6 +36,11 @@ public interface IDataProvider // LOCKED_SHAREDCACHE included. False where writers do not wait for the whole database. bool IsBusy(Exception exception); + // Whether the database has one writer at a time, which a write transaction keeps from its beginning to its end, so that + // every write waits for every other connection's write transaction, whatever rows they touch. True on SQLite, whose + // timings of that wait the default lock keeps in core (DatabaseDistributedLock). + bool HasOneWriter { get; } + // A condition of a LINQ query: the text holds the fragment, as it is (LikePattern), without regard to case as the database // compares text. A match on a part of the text uses no index. Expression> ContainsIgnoringCase(string fragment); diff --git a/src/FluentCMS.Core/Storage/MigrationRunner.cs b/src/FluentCMS.Core/Storage/MigrationRunner.cs index 23ab34cf0..fd61ed680 100644 --- a/src/FluentCMS.Core/Storage/MigrationRunner.cs +++ b/src/FluentCMS.Core/Storage/MigrationRunner.cs @@ -1,4 +1,5 @@ using System.Text; +using FluentCMS.Core.Caching; using FluentCMS.Core.DependencyInjection; using FluentCMS.Core.Plugins; using FluentCMS.Providers; @@ -215,8 +216,9 @@ internal async Task LockContentSchemaAsync(CancellationT $"check the owner and lease_until of its row in {StorageNames.Lock}: a lease_until that keeps moving means another instance or a " + "'fluentcms schema apply' still holds it and is working, and one that stands still ahead of this instance's clock means its holder " + "stopped and the clocks of the instances are not in step; on SQLite one that stands still behind the clock can also mean that its " + - "holder is working inside a long write transaction, which keeps it from renewing until the transaction ends; once no instance " + - "is running, set the row's owner to null."); + "holder is working inside a long write transaction, which keeps it from renewing until the transaction ends; when the log has " + + $"event {LockLog.TimedOutBusyId} for the lock, it can also mean that the database was busy: another connection kept SQLite's one " + + "writer to the end of the wait, and the row may be free; once no instance is running, set the row's owner to null."); // The steps that bring the content tables up to the host's schema: schema sync's comparison, with renames and the // changes sync leaves undone. The applied snapshot tells which tables under a plugin's prefix are content tables. diff --git a/src/FluentCMS.Data.PostgreSql/PostgreSqlDataProvider.cs b/src/FluentCMS.Data.PostgreSql/PostgreSqlDataProvider.cs index 11ff0f2dc..357b3bad4 100644 --- a/src/FluentCMS.Data.PostgreSql/PostgreSqlDataProvider.cs +++ b/src/FluentCMS.Data.PostgreSql/PostgreSqlDataProvider.cs @@ -43,6 +43,8 @@ public bool IsDeadlock(Exception exception) // Writers wait for the rows they write, not for the whole database. public bool IsBusy(Exception exception) => false; + public bool HasOneWriter => false; + // Npgsql's migrations lock is a lock of the history table, which ends with the transaction that took it. public Task ReadMigrationsLockAsync(DbContext context, CancellationToken cancellationToken) => Task.FromResult(null); diff --git a/src/FluentCMS.Data.SqlServer/SqlServerDataProvider.cs b/src/FluentCMS.Data.SqlServer/SqlServerDataProvider.cs index 73fac7dcd..c0ab00797 100644 --- a/src/FluentCMS.Data.SqlServer/SqlServerDataProvider.cs +++ b/src/FluentCMS.Data.SqlServer/SqlServerDataProvider.cs @@ -44,6 +44,8 @@ public bool IsDeadlock(Exception exception) // Writers wait for the rows they write, not for the whole database. public bool IsBusy(Exception exception) => false; + public bool HasOneWriter => false; + // EF's migrations lock is an application lock of the session, which ends with it. public Task ReadMigrationsLockAsync(DbContext context, CancellationToken cancellationToken) => Task.FromResult(null); diff --git a/src/FluentCMS.Data.Sqlite/SqliteDataProvider.cs b/src/FluentCMS.Data.Sqlite/SqliteDataProvider.cs index 9cea08536..f188418d9 100644 --- a/src/FluentCMS.Data.Sqlite/SqliteDataProvider.cs +++ b/src/FluentCMS.Data.Sqlite/SqliteDataProvider.cs @@ -48,6 +48,8 @@ public bool IsUniqueViolation(Exception exception) => public bool IsBusy(Exception exception) => (exception as SqliteException ?? exception.InnerException as SqliteException) is { SqliteErrorCode: raw.SQLITE_BUSY or raw.SQLITE_LOCKED }; + public bool HasOneWriter => true; + // SQLite's LIKE folds the case of ASCII letters only. public Expression> ContainsIgnoringCase(string fragment) { diff --git a/src/FluentCMS.Sdk/Providers/IDistributedLock.cs b/src/FluentCMS.Sdk/Providers/IDistributedLock.cs index 66441796e..02decc0c7 100644 --- a/src/FluentCMS.Sdk/Providers/IDistributedLock.cs +++ b/src/FluentCMS.Sdk/Providers/IDistributedLock.cs @@ -11,7 +11,8 @@ public interface IDistributedLock /// The name of the lock, at most 128 characters. Names are a fixed set, such as one per task, not one per item. /// /// How long to wait while another holder has the lock: to try once, or - /// to wait until it is free. + /// to wait until it is free. An implementation may answer shortly after the + /// timeout while it confirms that a holder whose lease ran out has stopped; core's default does so for up to 5 seconds. /// /// Cancels the wait. /// The held lock, which disposing releases, or when the timeout passed first. diff --git a/tests/FluentCMS.Core.IntegrationTests/Caching/DistributedLockTests.cs b/tests/FluentCMS.Core.IntegrationTests/Caching/DistributedLockTests.cs index 431e0ed72..80e67d881 100644 --- a/tests/FluentCMS.Core.IntegrationTests/Caching/DistributedLockTests.cs +++ b/tests/FluentCMS.Core.IntegrationTests/Caching/DistributedLockTests.cs @@ -1,5 +1,7 @@ +using System.Data.Common; using System.Diagnostics; using FluentCMS.Core.Caching; +using FluentCMS.Core.IntegrationTests.Diagnostics; using FluentCMS.Core.IntegrationTests.Testing; using FluentCMS.Core.Storage; using FluentCMS.Core.Storage.Entities; @@ -7,7 +9,9 @@ using FluentCMS.Testing; using Microsoft.Data.Sqlite; using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Diagnostics; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; using Shouldly; using Xunit; @@ -322,6 +326,160 @@ public async Task AHolderIsToldWhenAnotherTookTheLockWhileAWriteTransactionKeeps } } + // A waiter that found the lease over while a write transaction kept the holder's renewal waiting gets the writer first + // when the transaction ends more often than not, since it began to wait last and SQLite sleeps less between its first + // tries. It takes nothing over then, and the renewal, which has been trying all along, renews the lease before the + // waiter's confirming try a second later: in a median of 80 milliseconds after the transaction in a review's 50 rounds, + // and at most 660. + [Fact] + public async Task AHolderKeepsTheLockWhenAWaiterFindsItsLeaseOverAtTheEndOfAWriteTransaction() + { + var takes = new TakeSignals(Name); + var logs = new LogCapture(); + await using var holderHost = Impatient(new FluentCmsTestHost().UseSqlite(_file)); + await using var waiterHost = Impatient(Watched(new FluentCmsTestHost().UseSqlite(_file), takes)); + var waiter = RealClockLease(waiterHost, TimeSpan.FromMilliseconds(300), logs); + await using var held = await RealClockShortLease(holderHost).TryAcquireAsync(Name, TimeSpan.Zero, Token); + held.ShouldNotBeNull(); + var holder = (await RowAsync(holderHost)).Owner; + + for (var round = 0; round < 10; round++) + { + using var stop = CancellationTokenSource.CreateLinkedTokenSource(Token); + Task waiting; + await using (var writer = await WriteTransactionAsync()) + { + takes.Begins.Forget(); + waiting = Task.Run(() => waiter.TryAcquireAsync(Name, Patience, stop.Token), Token); + await takes.Begins.NextAsync(1, Patience); + } + + (await LiveRowAsync(holderHost)).Owner.ShouldBe(holder, $"the waiter took the lock in round {round}"); + + // The tries of the waiter's confirmation, which find the lease renewed. + takes.Reads.Forget(); + await takes.Reads.NextAsync(3, Patience); + (await RowAsync(holderHost)).Owner.ShouldBe(holder, $"the waiter took the lock in round {round}"); + await stop.CancelAsync(); + (await CanceledAsync(waiting)).ShouldBeNull(); + } + + held.Lost.IsCancellationRequested.ShouldBeFalse(); + logs.Entries.ShouldNotContain(entry => entry.EventId.Id == LockLog.LeaseTakenOverId); + } + + // A holder that stopped leaves its lease over and as it was, so a waiter takes it over at the try a second after it saw + // it so: two tries that each began a write transaction. + [Fact] + public async Task AWaiterTakesOverTheLeaseOfAHolderThatStoppedOnceItsConfirmationFindsItAsItWas() + { + var takes = new TakeSignals(Name); + var logs = new LogCapture(); + await using var host = Watched(new FluentCmsTestHost().UseSqlite(_file), takes); + var locks = RealClockLease(host, TimeSpan.FromSeconds(30), logs); + await InsertAsync(host, Crashed, DateTimeOffset.UtcNow - TimeSpan.FromSeconds(1)); + + var watch = Stopwatch.StartNew(); + await using var taken = await locks.TryAcquireAsync(Name, Patience, Token); + + taken.ShouldNotBeNull(); + takes.Count.ShouldBeGreaterThanOrEqualTo(2); + watch.Elapsed.ShouldBeGreaterThan(TimeSpan.FromMilliseconds(900)); + var takeOver = logs.Entries.ShouldHaveSingleItem(); + takeOver.EventId.Id.ShouldBe(LockLog.LeaseTakenOverId); + takeOver.State["Holder"].ShouldBe(Crashed); + } + + // The export and the import try once, and take over the lock of a predecessor that stopped, a confirmation late. + [Fact] + public async Task ATryOnceTakesOverTheLeaseOfAHolderThatStoppedEveryTime() + { + var locks = RealClockLease(_first, TimeSpan.FromSeconds(30)); + await InsertAsync(_first, Crashed, DateTimeOffset.UtcNow - TimeSpan.FromSeconds(1)); + + for (var round = 0; round < 30; round++) + { + var watch = Stopwatch.StartNew(); + var taken = await locks.TryAcquireAsync(Name, TimeSpan.Zero, Token); + taken.ShouldNotBeNull($"round {round}, after {watch.Elapsed}"); + await taken.DisposeAsync(); + + await using var scope = _first.Services.CreateAsyncScope(); + await scope.ServiceProvider.GetRequiredService().Set() + .Where(row => row.Name == Name) + .ExecuteUpdateAsync(row => row.SetProperty(column => column.Owner, Crashed).SetProperty(column => column.LeaseUntil, DateTimeOffset.UtcNow - TimeSpan.FromSeconds(1)), Token); + } + } + + [Fact] + public async Task OfTwoWaitersOnTheLeaseOfAHolderThatStoppedOneTakesIt() + { + await InsertAsync(_first, Crashed, DateTimeOffset.UtcNow - TimeSpan.FromSeconds(1)); + var first = RealClockLease(_first, TimeSpan.FromSeconds(30)); + var second = RealClockLease(_second, TimeSpan.FromSeconds(30)); + using var stop = CancellationTokenSource.CreateLinkedTokenSource(Token); + + var waits = new[] + { + Task.Run(() => first.TryAcquireAsync(Name, Patience, stop.Token), Token), + Task.Run(() => second.TryAcquireAsync(Name, Patience, stop.Token), Token), + }; + + // The loser waits for the winner, which keeps the lock, so it is canceled once one has it. + var winner = await (await Task.WhenAny(waits)); + await stop.CancelAsync(); + var attempts = new[] { await CanceledAsync(waits[0]), await CanceledAsync(waits[1]) }; + + winner.ShouldNotBeNull(); + attempts.Count(held => held is not null).ShouldBe(1); + await winner.DisposeAsync(); + } + + // The holder releases the lock while a waiter that found its lease over waits for the writer, so the waiter takes a free + // lock, which no holder left behind. + [Fact] + public async Task AWaiterThatGetsALockReleasedWhileItWaitedForTheWriterLogsNoTakeOver() + { + var takes = new TakeSignals(Name); + var logs = new LogCapture(); + await using var holderHost = Impatient(new FluentCmsTestHost().UseSqlite(_file)); + await using var waiterHost = Impatient(Watched(new FluentCmsTestHost().UseSqlite(_file), takes)); + var waiter = RealClockLease(waiterHost, TimeSpan.FromMilliseconds(300), logs); + var held = await RealClockShortLease(holderHost).TryAcquireAsync(Name, TimeSpan.Zero, Token); + held.ShouldNotBeNull(); + + Task waiting; + Task releasing; + await using (var writer = await WriteTransactionAsync()) + { + waiting = Task.Run(() => waiter.TryAcquireAsync(Name, Patience, Token), Token); + await takes.Begins.NextAsync(1, Patience); + releasing = Task.Run(() => held.DisposeAsync().AsTask(), Token); + } + + await releasing.WaitAsync(Patience, Token); + await using var next = await waiting; + next.ShouldNotBeNull(); + logs.Entries.ShouldNotContain(entry => entry.EventId.Id == LockLog.LeaseTakenOverId); + } + + // The row is free, and another connection keeps the database's one writer for longer than the take waits. + [Fact] + public async Task AWaitThatTimesOutOnABusyDatabaseSaysSo() + { + await using var host = Impatient(new FluentCmsTestHost().UseSqlite(_file)); + var logs = new LogCapture(); + var locks = RealClockLease(host, TimeSpan.FromMilliseconds(300), logs); + await (await locks.TryAcquireAsync(Name, TimeSpan.Zero, Token)).ShouldNotBeNull().DisposeAsync(); + + await using (var writer = await WriteTransactionAsync()) + { + (await locks.TryAcquireAsync(Name, TimeSpan.Zero, Token)).ShouldBeNull(); + } + + logs.Entries.ShouldHaveSingleItem().EventId.Id.ShouldBe(LockLog.TimedOutBusyId); + } + [Fact] public async Task ABadNameOrTimeoutIsRefused() { @@ -371,14 +529,53 @@ public async Task TakesThatRaceOnTheFirstUseOfANameEndWithinTheirTimeout() // A lease of 300 milliseconds on the real clock, renewed every 100, for a test whose time passes in SQLite's waits. private static DatabaseDistributedLock RealClockShortLease(FluentCmsTestHost host) => RealClockLease(host, TimeSpan.FromMilliseconds(300)); - private static DatabaseDistributedLock RealClockLease(FluentCmsTestHost host, TimeSpan lease) => new( + private static DatabaseDistributedLock RealClockLease(FluentCmsTestHost host, TimeSpan lease, LogCapture? logs = null) => new( host.Services.GetRequiredService(), TimeProvider.System, - NullLogger.Instance) + logs is null ? NullLogger.Instance : new Logger(new LoggerFactory([logs]))) { Lease = lease, }; + // What a holder that stopped without releasing the lock leaves behind. + private static async Task InsertAsync(FluentCmsTestHost host, string owner, DateTimeOffset leaseUntil) + { + await using var scope = host.Services.CreateAsyncScope(); + var context = scope.ServiceProvider.GetRequiredService(); + context.Add(new LockRow { Name = Name, Owner = owner, AcquiredAt = leaseUntil - TimeSpan.FromSeconds(30), LeaseUntil = leaseUntil }); + await context.SaveChangesAsync(Token); + } + + // The row once its lease runs on the real clock, read every 10 milliseconds: renewed by its holder, or taken over. + private static async Task LiveRowAsync(FluentCmsTestHost host) + { + var watch = Stopwatch.StartNew(); + while (true) + { + var row = await RowAsync(host); + if (row.LeaseUntil > DateTimeOffset.UtcNow) + { + return row; + } + + watch.Elapsed.ShouldBeLessThan(Patience); + await Task.Delay(TimeSpan.FromMilliseconds(10), Token); + } + } + + // The handle of a wait that was canceled, which is null unless the wait took the lock first. + private static async Task CanceledAsync(Task waiting) + { + try + { + return await waiting; + } + catch (OperationCanceledException) + { + return null; + } + } + // A write transaction on a connection of its own, as the schema apply of another process holds one; disposing it ends it. private async Task WriteTransactionAsync() { @@ -418,8 +615,46 @@ private static async Task RowAsync(FluentCmsTestHost host) return await scope.ServiceProvider.GetRequiredService().Set().AsNoTracking().SingleAsync(row => row.Name == Name, Token); } - private static FluentCmsTestHost Watched(FluentCmsTestHost host, CommandSignals commands) => - host.Configure(cms => cms.Services.ConfigureDbContext(options => options.AddInterceptors(commands))); + private static FluentCmsTestHost Watched(FluentCmsTestHost host, IInterceptor signals) => + host.Configure(cms => cms.Services.ConfigureDbContext(options => options.AddInterceptors(signals))); + + // Signals a take that found the lock's lease over, or the lock free, as it begins its write transaction: a transaction + // that begins on the connection that last read the lock's row. + private sealed class TakeSignals(string name) : IDbCommandInterceptor, IDbTransactionInterceptor + { + private DbConnection? _read; + + private int _count; + + public CommandSignals.Signals Begins { get; } = new(); + + // Every read of the lock's row, each try's first among them. + public CommandSignals.Signals Reads { get; } = new(); + + public int Count => Volatile.Read(ref _count); + + public ValueTask ReaderExecutedAsync(DbCommand command, CommandExecutedEventData eventData, DbDataReader result, CancellationToken cancellationToken = default) + { + if (command.CommandText.Contains("\"cms_lock\"", StringComparison.Ordinal) && command.Parameters.Cast().Any(parameter => Equals(parameter.Value, name))) + { + Volatile.Write(ref _read, command.Connection); + Reads.Add(); + } + + return ValueTask.FromResult(result); + } + + public ValueTask> TransactionStartingAsync(DbConnection connection, TransactionStartingEventData eventData, InterceptionResult result, CancellationToken cancellationToken = default) + { + if (Interlocked.CompareExchange(ref _read, null, connection) == connection) + { + Interlocked.Increment(ref _count); + Begins.Add(); + } + + return ValueTask.FromResult(result); + } + } private static void InterlockedMax(ref int target, int value) { diff --git a/tests/FluentCMS.Data.Tests/Migrations/LostLock.cs b/tests/FluentCMS.Data.Tests/Migrations/LostLock.cs new file mode 100644 index 000000000..40ee4836e --- /dev/null +++ b/tests/FluentCMS.Data.Tests/Migrations/LostLock.cs @@ -0,0 +1,79 @@ +using System.Data.Common; +using FluentCMS.Core.DependencyInjection; +using FluentCMS.Core.Storage; +using FluentCMS.Providers; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Diagnostics; +using Microsoft.Extensions.DependencyInjection; + +namespace FluentCMS.Data.Tests.Migrations; + +// A lock that another holder takes from its holder during the work on the content tables: as soon as it is held, so before +// the operations are planned; as the second command that adds a column runs, after the first ran; or once the transaction +// of the operations has committed. Its handle's Lost is canceled then. +public sealed class LostLock(LostLock.Moment when) : IDistributedLock, IDisposable +{ + private readonly CancellationTokenSource _lost = new(); + private readonly Moment _when = when; + private int _altered; + + public enum Moment + { + BeforeThePlan, + WhileTheOperationsRun, + AfterTheOperations, + } + + // Registered before AddFluentCms, so that core's default lock gives way to it. + public void Lock(IServiceCollection services) => services.AddSingleton(this); + + public void Configure(FluentCmsBuilder cms) => + cms.Services.ConfigureDbContext(options => options.AddInterceptors(new Commands(this), new Transactions(this))); + + public void Dispose() => _lost.Dispose(); + + public async Task TryAcquireAsync(string name, TimeSpan timeout, CancellationToken cancellationToken = default) + { + if (_when == Moment.BeforeThePlan) + { + await _lost.CancelAsync(); + } + + return new Handle(name, _lost.Token); + } + + private sealed class Handle(string name, CancellationToken lost) : IDistributedLockHandle + { + public string Name => name; + + public CancellationToken Lost => lost; + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + } + + private sealed class Commands(LostLock owner) : DbCommandInterceptor + { + public override async ValueTask> NonQueryExecutingAsync(DbCommand command, CommandEventData eventData, InterceptionResult result, CancellationToken cancellationToken = default) + { + if (command.CommandText.TrimStart().StartsWith("ALTER TABLE", StringComparison.OrdinalIgnoreCase) + && Interlocked.Increment(ref owner._altered) == 2 + && owner._when == Moment.WhileTheOperationsRun) + { + await owner._lost.CancelAsync(); + } + + return result; + } + } + + private sealed class Transactions(LostLock owner) : DbTransactionInterceptor + { + public override async Task TransactionCommittedAsync(DbTransaction transaction, TransactionEndEventData eventData, CancellationToken cancellationToken = default) + { + if (owner._when == Moment.AfterTheOperations && Volatile.Read(ref owner._altered) > 0) + { + await owner._lost.CancelAsync(); + } + } + } +} diff --git a/tests/FluentCMS.Data.Tests/Migrations/SchemaApplyTests.cs b/tests/FluentCMS.Data.Tests/Migrations/SchemaApplyTests.cs index 52e8d17ad..f37c5371c 100644 --- a/tests/FluentCMS.Data.Tests/Migrations/SchemaApplyTests.cs +++ b/tests/FluentCMS.Data.Tests/Migrations/SchemaApplyTests.cs @@ -628,6 +628,39 @@ public async Task ARunWhoseTransactionOutlastsTheLeaseOfItsLockFinishes() _database.Columns("cms_article").ShouldContain("summary"); } + // The lock is lost, as when a waiter on SQLite took it at the end of a long transaction: before the plan, while the + // operations run once the first of them ran, or after they committed and before the history is written. + [Theory] + [InlineData(LostLock.Moment.BeforeThePlan, "FCMS3339: the lock fluentcms:schema-sync was lost", "while the operations were planned, so nothing was applied")] + [InlineData(LostLock.Moment.WhileTheOperationsRun, "FCMS3339: the lock fluentcms:schema-sync was lost", "while the operations were applied; their transaction was rolled back")] + [InlineData(LostLock.Moment.AfterTheOperations, "FCMS3340: the content tables match schema ", "the lock fluentcms:schema-sync was lost")] + public async Task ARunThatLosesItsLockSaysToRunAgain(LostLock.Moment when, string start, string middle) + { + await MigrateAndApplyAsync(ArticleSchemas.Titled); + var schema = ArticleSchemas.Article("title", "subtitle", "summary"); + var afterTheOperations = when == LostLock.Moment.AfterTheOperations; + using var lost = new LostLock(when); + + using (var host = _database.CreateHost(schema, Production, settings: NoMigrationOnStartup, services: lost.Lock, configure: lost.Configure)) + { + var (exitCode, _, error) = await RunAsync(host, "schema", "apply"); + + exitCode.ShouldBe(1); + error.ShouldStartWith(start); + error.ShouldContain(middle); + error.ShouldContain("run 'fluentcms schema apply' again"); + } + + History().ShouldHaveSingleItem(); + _database.Columns("cms_article").Contains("subtitle").ShouldBe(afterTheOperations); + + var (again, _, againError) = await RunAsync(schema, "schema", "apply"); + + again.ShouldBe(0, againError); + _database.Columns("cms_article").ShouldContain("summary"); + History().Count.ShouldBe(2); + } + [Theory] [InlineData(new[] { "--output", "schema.sql" }, "FCMS3300: --output names the file of --script, so it needs --script")] [InlineData(new[] { "--script", "--dry-run" }, "FCMS3300: --script and --dry-run both apply nothing; give one")] diff --git a/tests/FluentCMS.Data.Tests/Migrations/SchemaSyncTests.cs b/tests/FluentCMS.Data.Tests/Migrations/SchemaSyncTests.cs index 1ea7f2f91..ee36e1519 100644 --- a/tests/FluentCMS.Data.Tests/Migrations/SchemaSyncTests.cs +++ b/tests/FluentCMS.Data.Tests/Migrations/SchemaSyncTests.cs @@ -399,6 +399,7 @@ public async Task ASyncGivesUpOnALockThatIsNeverReleased(bool frozenWallClock) var exception = await Should.ThrowAsync(Runner(host).SyncContentSchemaAsync(ArticleSchemas.WithSubtitle, Token).WaitAsync(TimeSpan.FromSeconds(30), Token)); exception.Message.ShouldStartWith($"The lock {MigrationRunner.SchemaSyncLock} was not free within 15 minutes, so the content tables were not changed; check the owner and lease_until of its row in cms_lock"); + exception.Message.ShouldContain("when the log has event 9166 for the lock, it can also mean that the database was busy"); _database.Columns("cms_article").ShouldNotContain("subtitle"); } diff --git a/tests/FluentCMS.Data.Tests/Sqlite/SqliteFileDatabase.cs b/tests/FluentCMS.Data.Tests/Sqlite/SqliteFileDatabase.cs index b8f7b001d..51c6d6808 100644 --- a/tests/FluentCMS.Data.Tests/Sqlite/SqliteFileDatabase.cs +++ b/tests/FluentCMS.Data.Tests/Sqlite/SqliteFileDatabase.cs @@ -64,6 +64,8 @@ private sealed class UnflushedSqliteProvider(SqliteDataProvider inner) : IDataPr public bool IsBusy(Exception exception) => inner.IsBusy(exception); + public bool HasOneWriter => inner.HasOneWriter; + public Task ReadMigrationsLockAsync(DbContext context, CancellationToken cancellationToken) => inner.ReadMigrationsLockAsync(context, cancellationToken); public Expression> ContainsIgnoringCase(string fragment) => inner.ContainsIgnoringCase(fragment);